py-data-engine 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (200) hide show
  1. data_engine/__init__.py +37 -0
  2. data_engine/application/__init__.py +39 -0
  3. data_engine/application/actions.py +42 -0
  4. data_engine/application/catalog.py +151 -0
  5. data_engine/application/control.py +213 -0
  6. data_engine/application/details.py +73 -0
  7. data_engine/application/runtime.py +449 -0
  8. data_engine/application/workspace.py +62 -0
  9. data_engine/authoring/__init__.py +14 -0
  10. data_engine/authoring/builder.py +31 -0
  11. data_engine/authoring/execution/__init__.py +6 -0
  12. data_engine/authoring/execution/app.py +6 -0
  13. data_engine/authoring/execution/context.py +82 -0
  14. data_engine/authoring/execution/continuous.py +176 -0
  15. data_engine/authoring/execution/grouped.py +106 -0
  16. data_engine/authoring/execution/logging.py +83 -0
  17. data_engine/authoring/execution/polling.py +135 -0
  18. data_engine/authoring/execution/runner.py +210 -0
  19. data_engine/authoring/execution/single.py +171 -0
  20. data_engine/authoring/flow.py +361 -0
  21. data_engine/authoring/helpers.py +160 -0
  22. data_engine/authoring/model.py +59 -0
  23. data_engine/authoring/primitives.py +430 -0
  24. data_engine/authoring/services.py +42 -0
  25. data_engine/devtools/__init__.py +3 -0
  26. data_engine/devtools/project_ast_map.py +503 -0
  27. data_engine/docs/__init__.py +1 -0
  28. data_engine/docs/sphinx_source/_static/custom.css +13 -0
  29. data_engine/docs/sphinx_source/api.rst +42 -0
  30. data_engine/docs/sphinx_source/conf.py +37 -0
  31. data_engine/docs/sphinx_source/guides/app-runtime-and-workspaces.md +397 -0
  32. data_engine/docs/sphinx_source/guides/authoring-flow-modules.md +215 -0
  33. data_engine/docs/sphinx_source/guides/configuring-flows.md +185 -0
  34. data_engine/docs/sphinx_source/guides/core-concepts.md +208 -0
  35. data_engine/docs/sphinx_source/guides/database-methods.md +107 -0
  36. data_engine/docs/sphinx_source/guides/duckdb-helpers.md +462 -0
  37. data_engine/docs/sphinx_source/guides/flow-context.md +538 -0
  38. data_engine/docs/sphinx_source/guides/flow-methods.md +206 -0
  39. data_engine/docs/sphinx_source/guides/getting-started.md +271 -0
  40. data_engine/docs/sphinx_source/guides/project-inventory.md +5683 -0
  41. data_engine/docs/sphinx_source/guides/project-map.md +118 -0
  42. data_engine/docs/sphinx_source/guides/recipes.md +268 -0
  43. data_engine/docs/sphinx_source/index.rst +22 -0
  44. data_engine/domain/__init__.py +92 -0
  45. data_engine/domain/actions.py +69 -0
  46. data_engine/domain/catalog.py +128 -0
  47. data_engine/domain/details.py +214 -0
  48. data_engine/domain/diagnostics.py +56 -0
  49. data_engine/domain/errors.py +104 -0
  50. data_engine/domain/inspection.py +99 -0
  51. data_engine/domain/logs.py +118 -0
  52. data_engine/domain/operations.py +172 -0
  53. data_engine/domain/operator.py +72 -0
  54. data_engine/domain/runs.py +155 -0
  55. data_engine/domain/runtime.py +279 -0
  56. data_engine/domain/source_state.py +17 -0
  57. data_engine/domain/support.py +54 -0
  58. data_engine/domain/time.py +23 -0
  59. data_engine/domain/workspace.py +159 -0
  60. data_engine/flow_modules/__init__.py +1 -0
  61. data_engine/flow_modules/flow_module_compiler.py +179 -0
  62. data_engine/flow_modules/flow_module_loader.py +201 -0
  63. data_engine/helpers/__init__.py +25 -0
  64. data_engine/helpers/duckdb.py +705 -0
  65. data_engine/hosts/__init__.py +1 -0
  66. data_engine/hosts/daemon/__init__.py +23 -0
  67. data_engine/hosts/daemon/app.py +221 -0
  68. data_engine/hosts/daemon/bootstrap.py +69 -0
  69. data_engine/hosts/daemon/client.py +465 -0
  70. data_engine/hosts/daemon/commands.py +64 -0
  71. data_engine/hosts/daemon/composition.py +310 -0
  72. data_engine/hosts/daemon/constants.py +15 -0
  73. data_engine/hosts/daemon/entrypoints.py +97 -0
  74. data_engine/hosts/daemon/lifecycle.py +191 -0
  75. data_engine/hosts/daemon/manager.py +272 -0
  76. data_engine/hosts/daemon/ownership.py +126 -0
  77. data_engine/hosts/daemon/runtime_commands.py +188 -0
  78. data_engine/hosts/daemon/runtime_control.py +31 -0
  79. data_engine/hosts/daemon/server.py +84 -0
  80. data_engine/hosts/daemon/shared_state.py +147 -0
  81. data_engine/hosts/daemon/state_sync.py +101 -0
  82. data_engine/platform/__init__.py +1 -0
  83. data_engine/platform/identity.py +35 -0
  84. data_engine/platform/local_settings.py +146 -0
  85. data_engine/platform/theme.py +259 -0
  86. data_engine/platform/workspace_models.py +190 -0
  87. data_engine/platform/workspace_policy.py +333 -0
  88. data_engine/runtime/__init__.py +1 -0
  89. data_engine/runtime/file_watch.py +185 -0
  90. data_engine/runtime/ledger_models.py +116 -0
  91. data_engine/runtime/runtime_db.py +938 -0
  92. data_engine/runtime/shared_state.py +523 -0
  93. data_engine/services/__init__.py +49 -0
  94. data_engine/services/daemon.py +64 -0
  95. data_engine/services/daemon_state.py +40 -0
  96. data_engine/services/flow_catalog.py +102 -0
  97. data_engine/services/flow_execution.py +48 -0
  98. data_engine/services/ledger.py +85 -0
  99. data_engine/services/logs.py +65 -0
  100. data_engine/services/runtime_binding.py +105 -0
  101. data_engine/services/runtime_execution.py +126 -0
  102. data_engine/services/runtime_history.py +62 -0
  103. data_engine/services/settings.py +58 -0
  104. data_engine/services/shared_state.py +28 -0
  105. data_engine/services/theme.py +59 -0
  106. data_engine/services/workspace_provisioning.py +224 -0
  107. data_engine/services/workspaces.py +74 -0
  108. data_engine/ui/__init__.py +3 -0
  109. data_engine/ui/cli/__init__.py +19 -0
  110. data_engine/ui/cli/app.py +161 -0
  111. data_engine/ui/cli/commands_doctor.py +178 -0
  112. data_engine/ui/cli/commands_run.py +80 -0
  113. data_engine/ui/cli/commands_start.py +100 -0
  114. data_engine/ui/cli/commands_workspace.py +97 -0
  115. data_engine/ui/cli/dependencies.py +44 -0
  116. data_engine/ui/cli/parser.py +56 -0
  117. data_engine/ui/gui/__init__.py +25 -0
  118. data_engine/ui/gui/app.py +116 -0
  119. data_engine/ui/gui/bootstrap.py +487 -0
  120. data_engine/ui/gui/bootstrapper.py +140 -0
  121. data_engine/ui/gui/cache_models.py +23 -0
  122. data_engine/ui/gui/control_support.py +185 -0
  123. data_engine/ui/gui/controllers/__init__.py +6 -0
  124. data_engine/ui/gui/controllers/flows.py +439 -0
  125. data_engine/ui/gui/controllers/runtime.py +245 -0
  126. data_engine/ui/gui/dialogs/__init__.py +12 -0
  127. data_engine/ui/gui/dialogs/messages.py +88 -0
  128. data_engine/ui/gui/dialogs/previews.py +222 -0
  129. data_engine/ui/gui/helpers/__init__.py +62 -0
  130. data_engine/ui/gui/helpers/inspection.py +81 -0
  131. data_engine/ui/gui/helpers/lifecycle.py +112 -0
  132. data_engine/ui/gui/helpers/scroll.py +28 -0
  133. data_engine/ui/gui/helpers/theming.py +87 -0
  134. data_engine/ui/gui/icons/dark_light.svg +12 -0
  135. data_engine/ui/gui/icons/documentation.svg +1 -0
  136. data_engine/ui/gui/icons/failed.svg +3 -0
  137. data_engine/ui/gui/icons/group.svg +4 -0
  138. data_engine/ui/gui/icons/home.svg +2 -0
  139. data_engine/ui/gui/icons/manual.svg +2 -0
  140. data_engine/ui/gui/icons/poll.svg +2 -0
  141. data_engine/ui/gui/icons/schedule.svg +4 -0
  142. data_engine/ui/gui/icons/settings.svg +2 -0
  143. data_engine/ui/gui/icons/started.svg +3 -0
  144. data_engine/ui/gui/icons/success.svg +3 -0
  145. data_engine/ui/gui/icons/view-log.svg +3 -0
  146. data_engine/ui/gui/icons.py +50 -0
  147. data_engine/ui/gui/launcher.py +48 -0
  148. data_engine/ui/gui/presenters/__init__.py +72 -0
  149. data_engine/ui/gui/presenters/docs.py +140 -0
  150. data_engine/ui/gui/presenters/logs.py +58 -0
  151. data_engine/ui/gui/presenters/runtime_projection.py +29 -0
  152. data_engine/ui/gui/presenters/sidebar.py +88 -0
  153. data_engine/ui/gui/presenters/steps.py +148 -0
  154. data_engine/ui/gui/presenters/workspace.py +39 -0
  155. data_engine/ui/gui/presenters/workspace_binding.py +75 -0
  156. data_engine/ui/gui/presenters/workspace_settings.py +182 -0
  157. data_engine/ui/gui/preview_models.py +37 -0
  158. data_engine/ui/gui/render_support.py +241 -0
  159. data_engine/ui/gui/rendering/__init__.py +12 -0
  160. data_engine/ui/gui/rendering/artifacts.py +95 -0
  161. data_engine/ui/gui/rendering/icons.py +50 -0
  162. data_engine/ui/gui/runtime.py +47 -0
  163. data_engine/ui/gui/state_support.py +193 -0
  164. data_engine/ui/gui/support.py +214 -0
  165. data_engine/ui/gui/surface.py +209 -0
  166. data_engine/ui/gui/theme.py +720 -0
  167. data_engine/ui/gui/widgets/__init__.py +34 -0
  168. data_engine/ui/gui/widgets/config.py +41 -0
  169. data_engine/ui/gui/widgets/logs.py +62 -0
  170. data_engine/ui/gui/widgets/panels.py +507 -0
  171. data_engine/ui/gui/widgets/sidebar.py +130 -0
  172. data_engine/ui/gui/widgets/steps.py +84 -0
  173. data_engine/ui/tui/__init__.py +5 -0
  174. data_engine/ui/tui/app.py +222 -0
  175. data_engine/ui/tui/bootstrap.py +475 -0
  176. data_engine/ui/tui/bootstrapper.py +117 -0
  177. data_engine/ui/tui/controllers/__init__.py +6 -0
  178. data_engine/ui/tui/controllers/flows.py +349 -0
  179. data_engine/ui/tui/controllers/runtime.py +167 -0
  180. data_engine/ui/tui/runtime.py +34 -0
  181. data_engine/ui/tui/state_support.py +141 -0
  182. data_engine/ui/tui/support.py +63 -0
  183. data_engine/ui/tui/theme.py +204 -0
  184. data_engine/ui/tui/widgets.py +123 -0
  185. data_engine/views/__init__.py +109 -0
  186. data_engine/views/actions.py +80 -0
  187. data_engine/views/artifacts.py +58 -0
  188. data_engine/views/flow_display.py +69 -0
  189. data_engine/views/logs.py +54 -0
  190. data_engine/views/models.py +96 -0
  191. data_engine/views/presentation.py +133 -0
  192. data_engine/views/runs.py +62 -0
  193. data_engine/views/state.py +39 -0
  194. data_engine/views/status.py +13 -0
  195. data_engine/views/text.py +109 -0
  196. py_data_engine-0.1.0.dist-info/METADATA +330 -0
  197. py_data_engine-0.1.0.dist-info/RECORD +200 -0
  198. py_data_engine-0.1.0.dist-info/WHEEL +5 -0
  199. py_data_engine-0.1.0.dist-info/entry_points.txt +2 -0
  200. py_data_engine-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,172 @@
1
+ """Domain models for selected-flow operation and step session state."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass, field
6
+ from typing import TYPE_CHECKING
7
+
8
+ if TYPE_CHECKING:
9
+ from data_engine.domain.logs import RuntimeStepEvent
10
+
11
+
12
+ @dataclass(frozen=True)
13
+ class OperationRowState:
14
+ """Display-ready state for one step row."""
15
+
16
+ status: str = "idle"
17
+ started_at: float | None = None
18
+ elapsed_seconds: float | None = None
19
+
20
+ def started(self, *, now: float) -> "OperationRowState":
21
+ """Return the running state for one started step."""
22
+ return type(self)(status="running", started_at=now, elapsed_seconds=None)
23
+
24
+ def finished(self, *, status: str, elapsed_seconds: float | None) -> "OperationRowState":
25
+ """Return the completed state for one finished step."""
26
+ return type(self)(status=status, started_at=None, elapsed_seconds=elapsed_seconds)
27
+
28
+ def normalized(self) -> "OperationRowState":
29
+ """Return the normalized post-success idle state."""
30
+ if self.status != "success":
31
+ return self
32
+ return type(self)(status="idle", started_at=None, elapsed_seconds=self.elapsed_seconds)
33
+
34
+ def duration_text(self, *, now: float, formatter) -> str:
35
+ """Return the formatted visible duration for this row."""
36
+ if self.status == "running" and isinstance(self.started_at, (int, float)):
37
+ return formatter(now - float(self.started_at))
38
+ if isinstance(self.elapsed_seconds, (int, float)):
39
+ return formatter(float(self.elapsed_seconds))
40
+ return ""
41
+
42
+
43
+ @dataclass(frozen=True)
44
+ class OperationFlowState:
45
+ """Tracked step state for one flow."""
46
+
47
+ current_index: int | None = None
48
+ rows: dict[str, OperationRowState] = field(default_factory=dict)
49
+
50
+ @classmethod
51
+ def from_operation_names(cls, operation_names: tuple[str, ...]) -> "OperationFlowState":
52
+ """Return the reset operation state for one flow."""
53
+ return cls(
54
+ current_index=None,
55
+ rows={operation_name: OperationRowState() for operation_name in operation_names},
56
+ )
57
+
58
+ def row_state(self, operation_name: str) -> OperationRowState | None:
59
+ """Return one row state by operation name."""
60
+ return self.rows.get(operation_name)
61
+
62
+ def apply_event(
63
+ self,
64
+ operation_names: tuple[str, ...],
65
+ event: "RuntimeStepEvent",
66
+ *,
67
+ now: float,
68
+ ) -> tuple["OperationFlowState", int | None]:
69
+ """Return updated flow step state after one runtime step event."""
70
+ if event.step_name is None or event.step_name not in operation_names:
71
+ return self, None
72
+ current = self.row_state(event.step_name) or OperationRowState()
73
+ rows = dict(self.rows)
74
+ if event.status == "started":
75
+ rows[event.step_name] = current.started(now=now)
76
+ return type(self)(current_index=operation_names.index(event.step_name), rows=rows), None
77
+ if event.status == "success":
78
+ rows[event.step_name] = current.finished(status="success", elapsed_seconds=event.elapsed_seconds)
79
+ index = operation_names.index(event.step_name)
80
+ return type(self)(current_index=index, rows=rows), index
81
+ if event.status == "failed":
82
+ rows[event.step_name] = current.finished(status="failed", elapsed_seconds=event.elapsed_seconds)
83
+ return type(self)(current_index=self.current_index, rows=rows), None
84
+ return self, None
85
+
86
+ def normalized_completed(self) -> "OperationFlowState":
87
+ """Return a copy with completed success rows reset to idle."""
88
+ return type(self)(
89
+ current_index=self.current_index,
90
+ rows={name: row.normalized() for name, row in self.rows.items()},
91
+ )
92
+
93
+
94
+ @dataclass(frozen=True)
95
+ class OperationSessionState:
96
+ """Tracked step state for all loaded flows."""
97
+
98
+ flow_states: dict[str, OperationFlowState] = field(default_factory=dict)
99
+
100
+ @classmethod
101
+ def empty(cls) -> "OperationSessionState":
102
+ """Return the empty operation session state."""
103
+ return cls()
104
+
105
+ def state_for(self, flow_name: str) -> OperationFlowState | None:
106
+ """Return one flow operation state by flow name."""
107
+ return self.flow_states.get(flow_name)
108
+
109
+ def reset_flow(self, flow_name: str, operation_names: tuple[str, ...]) -> "OperationSessionState":
110
+ """Return a copy with one flow reset to idle operation state."""
111
+ states = dict(self.flow_states)
112
+ states[flow_name] = OperationFlowState.from_operation_names(operation_names)
113
+ return type(self)(flow_states=states)
114
+
115
+ def ensure_flow(self, flow_name: str, operation_names: tuple[str, ...]) -> "OperationSessionState":
116
+ """Return a copy guaranteed to contain one flow state."""
117
+ state = self.state_for(flow_name)
118
+ if state is None or not state.rows:
119
+ return self.reset_flow(flow_name, operation_names)
120
+ return self
121
+
122
+ def row_state(self, flow_name: str, operation_name: str) -> OperationRowState | None:
123
+ """Return one operation row state from the current session."""
124
+ state = self.state_for(flow_name)
125
+ if state is None:
126
+ return None
127
+ return state.row_state(operation_name)
128
+
129
+ def apply_event(
130
+ self,
131
+ flow_name: str,
132
+ operation_names: tuple[str, ...],
133
+ event: "RuntimeStepEvent",
134
+ *,
135
+ now: float,
136
+ ) -> tuple["OperationSessionState", int | None]:
137
+ """Return updated session state after one runtime step event."""
138
+ if event.step_name is None or event.step_name not in operation_names:
139
+ return self, None
140
+ ensured = self.ensure_flow(flow_name, operation_names)
141
+ flow_state = ensured.state_for(flow_name) or OperationFlowState.from_operation_names(operation_names)
142
+ updated_flow_state, flash_index = flow_state.apply_event(operation_names, event, now=now)
143
+ states = dict(ensured.flow_states)
144
+ states[flow_name] = updated_flow_state
145
+ return type(self)(flow_states=states), flash_index
146
+
147
+ def duration_text(self, flow_name: str, operation_name: str, *, now: float, formatter) -> str:
148
+ """Return one formatted duration string for the selected step row."""
149
+ row_state = self.row_state(flow_name, operation_name)
150
+ if row_state is None:
151
+ return ""
152
+ return row_state.duration_text(now=now, formatter=formatter)
153
+
154
+ def normalize_completed(self, flow_name: str) -> "OperationSessionState":
155
+ """Return a copy with completed success rows normalized for one flow."""
156
+ state = self.state_for(flow_name)
157
+ if state is None:
158
+ return self
159
+ states = dict(self.flow_states)
160
+ states[flow_name] = state.normalized_completed()
161
+ return type(self)(flow_states=states)
162
+
163
+ def reset(self) -> "OperationSessionState":
164
+ """Return the empty operation session state."""
165
+ return type(self).empty()
166
+
167
+
168
+ __all__ = [
169
+ "OperationFlowState",
170
+ "OperationRowState",
171
+ "OperationSessionState",
172
+ ]
@@ -0,0 +1,72 @@
1
+ """Domain models for top-level operator session state."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass, replace
6
+ from pathlib import Path
7
+ from typing import TYPE_CHECKING
8
+
9
+ from data_engine.domain.catalog import FlowCatalogState
10
+ from data_engine.domain.operations import OperationSessionState
11
+ from data_engine.domain.runtime import RuntimeSessionState, WorkspaceControlState
12
+ from data_engine.domain.support import WorkspaceSupportState
13
+ from data_engine.domain.workspace import WorkspaceSessionState
14
+
15
+ if TYPE_CHECKING:
16
+ from data_engine.platform.workspace_models import WorkspacePaths
17
+
18
+
19
+ @dataclass(frozen=True)
20
+ class OperatorSessionState:
21
+ """Top-level operator state shared by one surface shell."""
22
+
23
+ workspace: WorkspaceSessionState
24
+ workspace_control: WorkspaceControlState
25
+ runtime: RuntimeSessionState
26
+ catalog: FlowCatalogState
27
+ operations: OperationSessionState
28
+ support: WorkspaceSupportState
29
+
30
+ @classmethod
31
+ def from_paths(
32
+ cls,
33
+ workspace_paths: "WorkspacePaths",
34
+ *,
35
+ override_root: Path | None = None,
36
+ ) -> "OperatorSessionState":
37
+ """Return the default operator state for one resolved workspace binding."""
38
+ return cls(
39
+ workspace=WorkspaceSessionState.from_paths(workspace_paths, override_root=override_root),
40
+ workspace_control=WorkspaceControlState.empty(),
41
+ runtime=RuntimeSessionState.empty(),
42
+ catalog=FlowCatalogState.empty(),
43
+ operations=OperationSessionState.empty(),
44
+ support=WorkspaceSupportState.empty(),
45
+ )
46
+
47
+ def with_workspace(self, workspace: WorkspaceSessionState) -> "OperatorSessionState":
48
+ """Return a copy with workspace session state replaced."""
49
+ return replace(self, workspace=workspace)
50
+
51
+ def with_workspace_control(self, workspace_control: WorkspaceControlState) -> "OperatorSessionState":
52
+ """Return a copy with workspace control state replaced."""
53
+ return replace(self, workspace_control=workspace_control)
54
+
55
+ def with_runtime(self, runtime: RuntimeSessionState) -> "OperatorSessionState":
56
+ """Return a copy with runtime session state replaced."""
57
+ return replace(self, runtime=runtime)
58
+
59
+ def with_catalog(self, catalog: FlowCatalogState) -> "OperatorSessionState":
60
+ """Return a copy with flow catalog state replaced."""
61
+ return replace(self, catalog=catalog)
62
+
63
+ def with_operations(self, operations: OperationSessionState) -> "OperatorSessionState":
64
+ """Return a copy with operation session state replaced."""
65
+ return replace(self, operations=operations)
66
+
67
+ def with_support(self, support: WorkspaceSupportState) -> "OperatorSessionState":
68
+ """Return a copy with support state replaced."""
69
+ return replace(self, support=support)
70
+
71
+
72
+ __all__ = ["OperatorSessionState"]
@@ -0,0 +1,155 @@
1
+ """Domain models for grouped flow-run state."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass
6
+ from datetime import UTC, datetime
7
+
8
+ from data_engine.domain.logs import FlowLogEntry
9
+
10
+ RunKey = tuple[str, str]
11
+
12
+
13
+ @dataclass(frozen=True)
14
+ class RunStepState:
15
+ """One collapsed step state inside a grouped run."""
16
+
17
+ step_name: str
18
+ status: str
19
+ elapsed_seconds: float | None
20
+ entry: FlowLogEntry
21
+
22
+
23
+ @dataclass(frozen=True)
24
+ class FlowRunState:
25
+ """One grouped run plus its raw log history."""
26
+
27
+ key: RunKey
28
+ display_label: str
29
+ source_label: str
30
+ status: str
31
+ elapsed_seconds: float | None
32
+ summary_entry: FlowLogEntry | None
33
+ steps: tuple[RunStepState, ...]
34
+ entries: tuple[FlowLogEntry, ...]
35
+
36
+ @classmethod
37
+ def group_entries(cls, entries: tuple[FlowLogEntry, ...]) -> tuple["FlowRunState", ...]:
38
+ """Group flow log entries into one state object per run."""
39
+ if not entries:
40
+ return ()
41
+
42
+ run_index_by_id: dict[str, int] = {}
43
+ mutable_runs: list[dict[str, object]] = []
44
+
45
+ for entry in entries:
46
+ event = entry.event
47
+ if event is None or event.run_id is None:
48
+ continue
49
+
50
+ source_label = event.source_label
51
+ run_token = event.run_id
52
+ if event.step_name is None:
53
+ run_index = run_index_by_id.get(run_token)
54
+ if run_index is None:
55
+ mutable_runs.append(
56
+ {
57
+ "key": (event.flow_name, run_token),
58
+ "source_label": source_label,
59
+ "status": event.status,
60
+ "elapsed_seconds": event.elapsed_seconds,
61
+ "summary_entry": entry,
62
+ "steps": [],
63
+ "entries": [entry],
64
+ }
65
+ )
66
+ run_index_by_id[run_token] = len(mutable_runs) - 1
67
+ else:
68
+ mutable_run = mutable_runs[run_index]
69
+ if event.status not in {"success", "finished"} or mutable_run["status"] not in {"failed", "stopped"}:
70
+ mutable_run["status"] = event.status
71
+ mutable_run["summary_entry"] = entry
72
+ mutable_run["elapsed_seconds"] = event.elapsed_seconds
73
+ run_entries = mutable_run["entries"]
74
+ assert isinstance(run_entries, list)
75
+ run_entries.append(entry)
76
+ continue
77
+
78
+ run_index = run_index_by_id.get(run_token)
79
+ if run_index is None:
80
+ mutable_runs.append(
81
+ {
82
+ "key": (event.flow_name, run_token),
83
+ "source_label": source_label,
84
+ "status": "started",
85
+ "elapsed_seconds": None,
86
+ "summary_entry": None,
87
+ "steps": [],
88
+ "entries": [],
89
+ }
90
+ )
91
+ run_index = len(mutable_runs) - 1
92
+ run_index_by_id[run_token] = run_index
93
+
94
+ mutable_run = mutable_runs[run_index]
95
+ steps = mutable_run["steps"]
96
+ assert isinstance(steps, list)
97
+ run_entries = mutable_run["entries"]
98
+ assert isinstance(run_entries, list)
99
+ run_entries.append(entry)
100
+ step_key = (event.flow_name, event.step_name, source_label)
101
+ step_state = RunStepState(
102
+ step_name=event.step_name,
103
+ status=event.status,
104
+ elapsed_seconds=event.elapsed_seconds,
105
+ entry=entry,
106
+ )
107
+ if event.status in {"success", "failed", "stopped"}:
108
+ for index in range(len(steps) - 1, -1, -1):
109
+ candidate = steps[index]
110
+ if (
111
+ isinstance(candidate, RunStepState)
112
+ and candidate.entry.event is not None
113
+ and candidate.entry.event.step_name is not None
114
+ and (
115
+ candidate.entry.event.flow_name,
116
+ candidate.entry.event.step_name,
117
+ candidate.entry.event.source_label,
118
+ )
119
+ == step_key
120
+ and candidate.entry.event.status == "started"
121
+ ):
122
+ steps[index] = step_state
123
+ break
124
+ else:
125
+ steps.append(step_state)
126
+ else:
127
+ steps.append(step_state)
128
+
129
+ return tuple(
130
+ cls(
131
+ key=run["key"],
132
+ display_label=cls._display_label_for_run(run),
133
+ source_label=run["source_label"],
134
+ status=run["status"],
135
+ elapsed_seconds=run["elapsed_seconds"],
136
+ summary_entry=run["summary_entry"],
137
+ steps=tuple(run["steps"]),
138
+ entries=tuple(run["entries"]),
139
+ )
140
+ for run in mutable_runs
141
+ )
142
+
143
+ @staticmethod
144
+ def _display_label_for_run(run: dict[str, object]) -> str:
145
+ summary_entry = run.get("summary_entry")
146
+ if isinstance(summary_entry, FlowLogEntry):
147
+ created_at = summary_entry.created_at_utc
148
+ else:
149
+ run_entries = run.get("entries")
150
+ first_entry = run_entries[0] if isinstance(run_entries, list) and run_entries else None
151
+ created_at = first_entry.created_at_utc if isinstance(first_entry, FlowLogEntry) else datetime.now(UTC)
152
+ return created_at.astimezone().strftime("%Y-%m-%d %I:%M:%S %p")
153
+
154
+
155
+ __all__ = ["FlowRunState", "RunKey", "RunStepState"]
@@ -0,0 +1,279 @@
1
+ """Domain models for operator runtime and control state."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from datetime import UTC, datetime, timedelta
6
+ from dataclasses import dataclass, replace
7
+ from enum import Enum
8
+ from typing import TYPE_CHECKING, Iterable, Mapping
9
+
10
+ from data_engine.domain.catalog import FlowCatalogLike
11
+ from data_engine.domain.time import parse_utc_text
12
+
13
+ CONTROL_CHECKPOINT_INTERVAL_SECONDS = 30.0
14
+ CONTROL_STALE_AFTER_SECONDS = 90.0
15
+
16
+ if TYPE_CHECKING:
17
+ from data_engine.hosts.daemon.manager import WorkspaceDaemonSnapshot
18
+
19
+
20
+ class DaemonLifecyclePolicy(str, Enum):
21
+ """Lifecycle ownership policy for one daemon instance."""
22
+
23
+ EPHEMERAL = "ephemeral"
24
+ PERSISTENT = "persistent"
25
+
26
+ @classmethod
27
+ def coerce(cls, value: object) -> "DaemonLifecyclePolicy":
28
+ """Normalize a raw lifecycle-policy input."""
29
+ if isinstance(value, cls):
30
+ return value
31
+ text = str(value or "").strip().lower()
32
+ if not text:
33
+ return cls.PERSISTENT
34
+ return cls(text)
35
+
36
+
37
+ @dataclass(frozen=True)
38
+ class ManualRunState:
39
+ """One active manual run grouped by the owning flow group."""
40
+
41
+ group_name: str | None
42
+ flow_name: str
43
+
44
+
45
+ @dataclass(frozen=True)
46
+ class RuntimeSessionState:
47
+ """Runtime/control state shared by operator surfaces."""
48
+
49
+ workspace_owned: bool = True
50
+ leased_by_machine_id: str | None = None
51
+ runtime_active: bool = False
52
+ runtime_stopping: bool = False
53
+ active_runtime_flow_names: tuple[str, ...] = ()
54
+ manual_runs: tuple[ManualRunState, ...] = ()
55
+
56
+ @classmethod
57
+ def empty(cls) -> "RuntimeSessionState":
58
+ """Return the default idle local-control runtime state."""
59
+ return cls()
60
+
61
+ @classmethod
62
+ def from_daemon_snapshot(
63
+ cls,
64
+ snapshot: "WorkspaceDaemonSnapshot",
65
+ flow_cards: Iterable[FlowCatalogLike],
66
+ ) -> "RuntimeSessionState":
67
+ """Build one normalized session state from a daemon snapshot and loaded flows."""
68
+ cards = tuple(flow_cards)
69
+ cards_by_name = {card.name: card for card in cards}
70
+ manual_runs = tuple(
71
+ ManualRunState(group_name=card.group, flow_name=flow_name)
72
+ for flow_name in snapshot.manual_runs
73
+ if (card := cards_by_name.get(flow_name)) is not None
74
+ )
75
+ active_runtime_flow_names = (
76
+ tuple(card.name for card in cards if card.valid and card.mode in {"poll", "schedule"})
77
+ if snapshot.runtime_active
78
+ else ()
79
+ )
80
+ return cls(
81
+ workspace_owned=snapshot.workspace_owned,
82
+ leased_by_machine_id=snapshot.leased_by_machine_id,
83
+ runtime_active=snapshot.runtime_active,
84
+ runtime_stopping=snapshot.runtime_stopping,
85
+ active_runtime_flow_names=active_runtime_flow_names,
86
+ manual_runs=manual_runs,
87
+ )
88
+
89
+ @property
90
+ def control_available(self) -> bool:
91
+ """Return whether the current workstation may issue control actions."""
92
+ return self.workspace_owned or self.leased_by_machine_id is None
93
+
94
+ @property
95
+ def manual_run_active(self) -> bool:
96
+ """Return whether any manual runs are currently active."""
97
+ return bool(self.manual_runs)
98
+
99
+ @property
100
+ def has_active_work(self) -> bool:
101
+ """Return whether engine or manual work is currently active."""
102
+ return self.runtime_active or self.manual_run_active
103
+
104
+ @property
105
+ def active_manual_runs(self) -> dict[str | None, str]:
106
+ """Return active manual runs keyed by flow group."""
107
+ return {run.group_name: run.flow_name for run in self.manual_runs}
108
+
109
+ def manual_flow_name_for_group(self, group_name: str | None) -> str | None:
110
+ """Return the active flow name for one group, if any."""
111
+ for run in self.manual_runs:
112
+ if run.group_name == group_name:
113
+ return run.flow_name
114
+ return None
115
+
116
+ def is_group_active(self, group_name: str, flow_groups_by_name: Mapping[str, str]) -> bool:
117
+ """Return whether a flow group is active through a manual run or engine run."""
118
+ if self.manual_flow_name_for_group(group_name) is not None:
119
+ return True
120
+ if not self.runtime_active:
121
+ return False
122
+ return any(flow_groups_by_name.get(flow_name) == group_name for flow_name in self.active_runtime_flow_names)
123
+
124
+ def with_manual_runs_map(self, active_manual_runs: Mapping[str | None, str]) -> "RuntimeSessionState":
125
+ """Return a copy with active manual runs replaced from a mapping."""
126
+ return replace(
127
+ self,
128
+ manual_runs=tuple(
129
+ ManualRunState(group_name=group_name, flow_name=flow_name)
130
+ for group_name, flow_name in active_manual_runs.items()
131
+ ),
132
+ )
133
+
134
+ def without_manual_group(self, group_name: str | None) -> "RuntimeSessionState":
135
+ """Return a copy with one manual-run group removed."""
136
+ return replace(
137
+ self,
138
+ manual_runs=tuple(run for run in self.manual_runs if run.group_name != group_name),
139
+ )
140
+
141
+ def with_active_runtime_flow_names(self, flow_names: Iterable[str]) -> "RuntimeSessionState":
142
+ """Return a copy with active engine-owned flow names replaced."""
143
+ return replace(self, active_runtime_flow_names=tuple(flow_names))
144
+
145
+ def with_runtime_flags(self, *, active: bool, stopping: bool) -> "RuntimeSessionState":
146
+ """Return a copy with updated engine active/stopping flags."""
147
+ return replace(self, runtime_active=active, runtime_stopping=stopping)
148
+
149
+ def reset(self) -> "RuntimeSessionState":
150
+ """Return the default idle state for a fresh workspace binding."""
151
+ return type(self).empty()
152
+
153
+
154
+ @dataclass(frozen=True)
155
+ class DaemonStatusState:
156
+ """Last normalized daemon status for one operator surface."""
157
+
158
+ workspace_owned: bool = True
159
+ leased_by_machine_id: str | None = None
160
+ engine_active: bool = False
161
+ engine_stopping: bool = False
162
+ manual_run_names: tuple[str, ...] = ()
163
+ last_checkpoint_at_utc: str | None = None
164
+ source: str = "none"
165
+
166
+ @classmethod
167
+ def empty(cls) -> "DaemonStatusState":
168
+ """Return the default no-daemon/no-lease status."""
169
+ return cls()
170
+
171
+ @classmethod
172
+ def from_snapshot(cls, snapshot: "WorkspaceDaemonSnapshot") -> "DaemonStatusState":
173
+ """Build one daemon-status value object from a daemon snapshot."""
174
+ return cls(
175
+ workspace_owned=snapshot.workspace_owned,
176
+ leased_by_machine_id=snapshot.leased_by_machine_id,
177
+ engine_active=snapshot.runtime_active,
178
+ engine_stopping=snapshot.runtime_stopping,
179
+ manual_run_names=tuple(snapshot.manual_runs),
180
+ last_checkpoint_at_utc=snapshot.last_checkpoint_at_utc,
181
+ source=snapshot.source,
182
+ )
183
+
184
+ def as_runtime_session(self, flow_cards: Iterable[FlowCatalogLike]) -> RuntimeSessionState:
185
+ """Project daemon status into runtime session state using the current loaded flows."""
186
+ return RuntimeSessionState.from_daemon_snapshot(
187
+ type("SnapshotProxy", (), {
188
+ "workspace_owned": self.workspace_owned,
189
+ "leased_by_machine_id": self.leased_by_machine_id,
190
+ "runtime_active": self.engine_active,
191
+ "runtime_stopping": self.engine_stopping,
192
+ "manual_runs": self.manual_run_names,
193
+ })(),
194
+ flow_cards,
195
+ )
196
+
197
+ @dataclass(frozen=True)
198
+ class WorkspaceControlState:
199
+ """Workspace lease/control state derived from one daemon snapshot."""
200
+
201
+ daemon_status: DaemonStatusState
202
+ control_status_text: str | None
203
+ blocked_status_text: str
204
+ local_request_pending: bool = False
205
+ takeover_remaining_seconds: int | None = None
206
+
207
+ @classmethod
208
+ def empty(cls) -> "WorkspaceControlState":
209
+ """Return the default no-daemon/no-lease control state."""
210
+ return cls(
211
+ daemon_status=DaemonStatusState.empty(),
212
+ control_status_text=None,
213
+ blocked_status_text="Takeover available.",
214
+ local_request_pending=False,
215
+ takeover_remaining_seconds=None,
216
+ )
217
+
218
+ @classmethod
219
+ def from_snapshot(
220
+ cls,
221
+ snapshot: "WorkspaceDaemonSnapshot",
222
+ *,
223
+ daemon_live: bool,
224
+ local_machine_id: str,
225
+ control_request: Mapping[str, object] | None = None,
226
+ daemon_startup_in_progress: bool = False,
227
+ now_utc: datetime | None = None,
228
+ ) -> "WorkspaceControlState":
229
+ """Build one workspace control state from the latest daemon snapshot."""
230
+ status = DaemonStatusState.from_snapshot(snapshot)
231
+ local_request_pending = (
232
+ isinstance(control_request, Mapping)
233
+ and str(control_request.get("requester_machine_id", "")).strip() == local_machine_id
234
+ )
235
+ checkpoint_at = parse_utc_text(status.last_checkpoint_at_utc) if status.last_checkpoint_at_utc else None
236
+ now = now_utc or datetime.now(UTC)
237
+
238
+ control_status_text: str | None
239
+ takeover_remaining_seconds: int | None = None
240
+
241
+ if status.source == "none":
242
+ control_status_text = None
243
+ elif status.workspace_owned:
244
+ if daemon_startup_in_progress:
245
+ control_status_text = "Trying to restore local control..."
246
+ elif checkpoint_at is None:
247
+ control_status_text = "This Workstation has control"
248
+ else:
249
+ age = max((now - checkpoint_at).total_seconds(), 0.0)
250
+ if age >= CONTROL_CHECKPOINT_INTERVAL_SECONDS and not daemon_live:
251
+ control_status_text = "Local engine is not responding"
252
+ else:
253
+ control_status_text = "This Workstation has control"
254
+ else:
255
+ owner = status.leased_by_machine_id or "Another machine"
256
+ if local_request_pending:
257
+ control_status_text = f"Control requested from {owner}"
258
+ elif checkpoint_at is None:
259
+ control_status_text = f"{owner} has control"
260
+ else:
261
+ stale_at = checkpoint_at + timedelta(seconds=CONTROL_STALE_AFTER_SECONDS)
262
+ takeover_remaining_seconds = max(int((stale_at - now).total_seconds()), 0)
263
+ if takeover_remaining_seconds <= 0:
264
+ control_status_text = "Takeover available"
265
+ else:
266
+ control_status_text = f"{owner} has control · takeover available in {takeover_remaining_seconds}s"
267
+
268
+ if status.leased_by_machine_id is None:
269
+ blocked_status_text = "Takeover available."
270
+ else:
271
+ blocked_status_text = f"{status.leased_by_machine_id} currently has control of this workspace."
272
+
273
+ return cls(
274
+ daemon_status=status,
275
+ control_status_text=control_status_text,
276
+ blocked_status_text=blocked_status_text,
277
+ local_request_pending=local_request_pending,
278
+ takeover_remaining_seconds=takeover_remaining_seconds,
279
+ )