terrine 0.1.0.dev0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (144) hide show
  1. terrine/__init__.py +120 -0
  2. terrine/_time.py +21 -0
  3. terrine/adapters/__init__.py +3 -0
  4. terrine/adapters/increment.py +365 -0
  5. terrine/adapters/opentelemetry.py +63 -0
  6. terrine/agent/__init__.py +6 -0
  7. terrine/agent/definitions.py +143 -0
  8. terrine/agent/design_schema.py +220 -0
  9. terrine/agent/policy.py +144 -0
  10. terrine/agent/schemas.py +285 -0
  11. terrine/agent/tools.py +956 -0
  12. terrine/backends/__init__.py +3 -0
  13. terrine/backends/capabilities.py +117 -0
  14. terrine/backends/dialect.py +55 -0
  15. terrine/backends/extrema.py +40 -0
  16. terrine/backends/joins.py +75 -0
  17. terrine/backends/numeric.py +657 -0
  18. terrine/backends/sequences.py +40 -0
  19. terrine/backends/timestamps.py +421 -0
  20. terrine/compile/__init__.py +3 -0
  21. terrine/compile/_asof.py +165 -0
  22. terrine/compile/_predicates.py +110 -0
  23. terrine/compile/anchors.py +492 -0
  24. terrine/compile/columns.py +1353 -0
  25. terrine/compile/cost.py +110 -0
  26. terrine/compile/dispatch.py +148 -0
  27. terrine/compile/layers.py +242 -0
  28. terrine/compile/materialize.py +91 -0
  29. terrine/compile/provenance.py +120 -0
  30. terrine/compile/ranges.py +91 -0
  31. terrine/compile/retention.py +127 -0
  32. terrine/compile/shapes/__init__.py +3 -0
  33. terrine/compile/shapes/_observations.py +502 -0
  34. terrine/compile/shapes/calendar_comparison.py +370 -0
  35. terrine/compile/shapes/cohorts.py +285 -0
  36. terrine/compile/shapes/event_panel.py +46 -0
  37. terrine/compile/shapes/event_sequence.py +498 -0
  38. terrine/compile/shapes/life_table.py +244 -0
  39. terrine/compile/shapes/ltv.py +287 -0
  40. terrine/compile/shapes/matched_t0.py +75 -0
  41. terrine/compile/shapes/sequential.py +320 -0
  42. terrine/compile/shapes/state_panel.py +460 -0
  43. terrine/compile/shapes/time_to_event.py +521 -0
  44. terrine/data.py +1311 -0
  45. terrine/dataset/__init__.py +5 -0
  46. terrine/dataset/export.py +463 -0
  47. terrine/dataset/manifest.py +254 -0
  48. terrine/dataset/reference.py +582 -0
  49. terrine/dataset/splits.py +373 -0
  50. terrine/dataset/training.py +114 -0
  51. terrine/dataset/transforms.py +182 -0
  52. terrine/errors.py +98 -0
  53. terrine/execution/__init__.py +5 -0
  54. terrine/execution/_bigquery.py +318 -0
  55. terrine/execution/_metadata.py +47 -0
  56. terrine/execution/_native.py +574 -0
  57. terrine/execution/_postgres.py +468 -0
  58. terrine/execution/_publish.py +299 -0
  59. terrine/execution/_snowflake.py +684 -0
  60. terrine/execution/backend.py +349 -0
  61. terrine/execution/gates.py +90 -0
  62. terrine/execution/inputs.py +923 -0
  63. terrine/export/__init__.py +31 -0
  64. terrine/export/diff_diff.py +177 -0
  65. terrine/export/lifelines.py +219 -0
  66. terrine/export/preflight.py +243 -0
  67. terrine/export/surrogate.py +168 -0
  68. terrine/frame.py +1042 -0
  69. terrine/model.py +397 -0
  70. terrine/orchestration/__init__.py +3 -0
  71. terrine/orchestration/airflow.py +115 -0
  72. terrine/orchestration/dagster.py +103 -0
  73. terrine/py.typed +0 -0
  74. terrine/recipes.py +295 -0
  75. terrine/runtime/__init__.py +70 -0
  76. terrine/runtime/api.py +803 -0
  77. terrine/runtime/arrow.py +98 -0
  78. terrine/runtime/backends/base.py +70 -0
  79. terrine/runtime/backends/duckdb.py +645 -0
  80. terrine/runtime/changes.py +71 -0
  81. terrine/runtime/cli.py +108 -0
  82. terrine/runtime/control.py +355 -0
  83. terrine/runtime/cost.py +211 -0
  84. terrine/runtime/events.py +194 -0
  85. terrine/runtime/execute.py +932 -0
  86. terrine/runtime/gc.py +43 -0
  87. terrine/runtime/graph.py +200 -0
  88. terrine/runtime/layers.py +61 -0
  89. terrine/runtime/leases.py +165 -0
  90. terrine/runtime/materialize.py +53 -0
  91. terrine/runtime/owner.py +57 -0
  92. terrine/runtime/planner.py +441 -0
  93. terrine/runtime/planning.py +394 -0
  94. terrine/runtime/saved_plan.py +337 -0
  95. terrine/runtime/snapshot.py +541 -0
  96. terrine/runtime/state.py +585 -0
  97. terrine/runtime/status.py +360 -0
  98. terrine/runtime/synchronization.py +15 -0
  99. terrine/semantic/__init__.py +3 -0
  100. terrine/semantic/bind.py +309 -0
  101. terrine/semantic/io.py +116 -0
  102. terrine/semantic/models.py +342 -0
  103. terrine/semantic/rules.py +113 -0
  104. terrine/semantic/sql.py +185 -0
  105. terrine/semantic/time.py +42 -0
  106. terrine/simulate/__init__.py +24 -0
  107. terrine/simulate/baseline.yaml +57 -0
  108. terrine/simulate/generate.py +182 -0
  109. terrine/simulate/scale.py +234 -0
  110. terrine/simulate/world.py +75 -0
  111. terrine/spec/__init__.py +96 -0
  112. terrine/spec/anchors.py +107 -0
  113. terrine/spec/dataset.py +260 -0
  114. terrine/spec/design.py +361 -0
  115. terrine/spec/expressions.py +440 -0
  116. terrine/spec/io.py +829 -0
  117. terrine/spec/panels.py +359 -0
  118. terrine/spec/sequence.py +165 -0
  119. terrine/spec/state_panel.py +246 -0
  120. terrine/spec/trajectory.py +144 -0
  121. terrine/spec/transforms.py +260 -0
  122. terrine/spec/validate.py +758 -0
  123. terrine/temporal/__init__.py +3 -0
  124. terrine/temporal/aggregate.py +688 -0
  125. terrine/temporal/attributes.py +389 -0
  126. terrine/temporal/daily.py +481 -0
  127. terrine/temporal/first.py +239 -0
  128. terrine/temporal/lookup.py +194 -0
  129. terrine/temporal/nonadditive.py +466 -0
  130. terrine/temporal/prefix.py +104 -0
  131. terrine/temporal/ranges.py +122 -0
  132. terrine/temporal/readiness.py +41 -0
  133. terrine/temporal/retention.py +225 -0
  134. terrine/temporal/sequence.py +900 -0
  135. terrine/temporal/sessionize.py +135 -0
  136. terrine/temporal/sketches.py +329 -0
  137. terrine/windows.py +54 -0
  138. terrine-0.1.0.dev0.dist-info/METADATA +164 -0
  139. terrine-0.1.0.dev0.dist-info/RECORD +144 -0
  140. terrine-0.1.0.dev0.dist-info/WHEEL +4 -0
  141. terrine-0.1.0.dev0.dist-info/entry_points.txt +2 -0
  142. terrine-0.1.0.dev0.dist-info/licenses/LICENSE +202 -0
  143. terrine-0.1.0.dev0.dist-info/licenses/NOTICE +4 -0
  144. terrine-0.1.0.dev0.dist-info/licenses/THIRD_PARTY_NOTICES.txt +219 -0
@@ -0,0 +1,168 @@
1
+ """Portable Arrow projection for downstream dataset consumers."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ from collections.abc import Mapping
7
+ from dataclasses import dataclass
8
+ from types import MappingProxyType
9
+ from typing import TYPE_CHECKING
10
+
11
+ from terrine.errors import RefusalSpec
12
+
13
+ if TYPE_CHECKING:
14
+ import pyarrow as pa
15
+
16
+ from terrine.data import MaterializedFrame
17
+
18
+ __all__ = ["ArrowSurrogate", "to_surrogate_arrow"]
19
+
20
+ _INVALID = RefusalSpec(
21
+ "terrine.export.invalid_surrogate",
22
+ "Surrogate export input is invalid ({reason}).",
23
+ )
24
+ _SURROGATE_METADATA_KEY = b"terrine.surrogate_arrow.v1"
25
+
26
+
27
+ @dataclass(frozen=True, slots=True)
28
+ class ArrowSurrogate:
29
+ """Versioned Arrow surrogate projection and portable metadata."""
30
+
31
+ data: pa.Table
32
+ metadata: Mapping[str, object]
33
+
34
+ def __post_init__(self) -> None:
35
+ import pyarrow as pa
36
+
37
+ if not isinstance(self.data, pa.Table):
38
+ raise TypeError("data must be a pyarrow.Table")
39
+ object.__setattr__(self, "metadata", MappingProxyType(dict(self.metadata)))
40
+
41
+ @classmethod
42
+ def from_arrow(cls, table: pa.Table) -> ArrowSurrogate:
43
+ raw = (table.schema.metadata or {}).get(_SURROGATE_METADATA_KEY)
44
+ if raw is None:
45
+ raise ValueError("Arrow schema has no Terrine surrogate metadata")
46
+ metadata = json.loads(raw.decode("utf-8"))
47
+ if metadata.get("schema_version") != 1:
48
+ raise ValueError("unsupported Terrine surrogate metadata version")
49
+ clean = dict(table.schema.metadata or {})
50
+ del clean[_SURROGATE_METADATA_KEY]
51
+ return cls(table.replace_schema_metadata(clean or None), metadata)
52
+
53
+ def to_arrow(self) -> pa.Table:
54
+ metadata = json.dumps(
55
+ dict(self.metadata), sort_keys=True, separators=(",", ":"), allow_nan=False
56
+ ).encode("utf-8")
57
+ schema_metadata = dict(self.data.schema.metadata or {})
58
+ schema_metadata[_SURROGATE_METADATA_KEY] = metadata
59
+ return self.data.replace_schema_metadata(schema_metadata)
60
+
61
+
62
+ def to_surrogate_arrow(
63
+ data: MaterializedFrame, *, features: tuple[str, ...], outcomes: tuple[str, ...]
64
+ ) -> ArrowSurrogate:
65
+ """Project Arrow training data with identity, missingness and provenance."""
66
+ from terrine.data import MaterializedFrame
67
+
68
+ if not isinstance(data, MaterializedFrame):
69
+ raise _INVALID.error(reason="materialized_frame_required")
70
+ table = data.table
71
+ structural = {
72
+ "time_to_event": ("entry_time",),
73
+ "ltv": ("age",),
74
+ "cohort_table": ("age", "at_risk", "observable"),
75
+ "life_table": ("time_bin", "censored_by_reason"),
76
+ }.get(data.provenance.shape, ())
77
+ requested = (*features, *outcomes)
78
+ if len(set(requested)) != len(requested):
79
+ raise _INVALID.error(reason="duplicate_surrogate_column")
80
+ if any(name not in table.column_names for name in requested):
81
+ raise _INVALID.error(reason="missing_surrogate_column")
82
+ identity = tuple(
83
+ dict.fromkeys(
84
+ name for name in (data.roles.unit, data.roles.anchor, data.roles.t0) if name is not None
85
+ )
86
+ )
87
+ role_support = tuple(
88
+ name
89
+ for name in (
90
+ data.roles.trial,
91
+ data.roles.assignment_probability,
92
+ data.roles.target_weight,
93
+ data.roles.trial_weight,
94
+ )
95
+ if name is not None
96
+ )
97
+ metadata_columns = tuple(
98
+ dict.fromkeys((*requested, *data.roles.outcomes, *structural, *identity, *role_support))
99
+ )
100
+ support = list(role_support)
101
+ for name in metadata_columns:
102
+ timing = data.provenance.expression_timings.get(name)
103
+ if timing is not None:
104
+ support.extend(
105
+ column
106
+ for column in (
107
+ timing.point_time_column,
108
+ timing.start_bound_column,
109
+ timing.end_bound_column,
110
+ )
111
+ if column is not None
112
+ )
113
+ meta = data.columns.get(name)
114
+ if meta is not None:
115
+ support.extend(
116
+ column
117
+ for column in (
118
+ meta.observed_from_column,
119
+ meta.observed_until_column,
120
+ meta.complete_column,
121
+ meta.reason_column,
122
+ )
123
+ if column is not None
124
+ )
125
+ retained = tuple(
126
+ dict.fromkeys((*identity, *requested, *data.roles.outcomes, *structural, *support))
127
+ )
128
+ missing = [name for name in retained if name not in table.column_names]
129
+ if missing:
130
+ raise _INVALID.error(reason=f"missing_surrogate_support_{missing[0]}")
131
+ projected = table.select(retained)
132
+ serialized = data.to_metadata()
133
+ role_metadata = dict(serialized["roles"])
134
+ projected_outcomes = [
135
+ name for name in (*outcomes, *data.roles.outcomes, *structural) if name in retained
136
+ ]
137
+ role_metadata["features"] = [name for name in features if name in retained]
138
+ role_metadata["outcomes"] = list(dict.fromkeys(projected_outcomes))
139
+ for role in ("unit", "anchor", "t0", "treatment", "weight", "cluster_unit", "prior_state"):
140
+ if role in role_metadata and role_metadata[role] not in retained:
141
+ role_metadata[role] = None
142
+ provenance = serialized["provenance"]
143
+ for field in ("expression_timings", "feature_cutoffs"):
144
+ provenance[field] = {
145
+ name: value for name, value in provenance[field].items() if name in retained
146
+ }
147
+ metadata = {
148
+ "schema_version": 1,
149
+ "design_hash": data.provenance.design_hash,
150
+ "model_hash": data.provenance.model_hash,
151
+ "source_generations": dict(data.provenance.source_generations),
152
+ "execution_snapshot": serialized["provenance"]["execution_snapshot"],
153
+ "provenance": provenance,
154
+ "roles": role_metadata,
155
+ "feature_columns": list(features),
156
+ "outcome_columns": list(dict.fromkeys(projected_outcomes)),
157
+ "support_columns": list(dict.fromkeys(support)),
158
+ "columns": {
159
+ name: value for name, value in serialized["columns"].items() if name in retained
160
+ },
161
+ "administrative_cutoff": (
162
+ data.provenance.administrative_cutoff.isoformat()
163
+ if data.provenance.administrative_cutoff is not None
164
+ else None
165
+ ),
166
+ "cohort_operation": serialized["provenance"].get("cohort_operation"),
167
+ }
168
+ return ArrowSurrogate(projected, metadata)