py-data-engine 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data_engine/__init__.py +37 -0
- data_engine/application/__init__.py +39 -0
- data_engine/application/actions.py +42 -0
- data_engine/application/catalog.py +151 -0
- data_engine/application/control.py +213 -0
- data_engine/application/details.py +73 -0
- data_engine/application/runtime.py +449 -0
- data_engine/application/workspace.py +62 -0
- data_engine/authoring/__init__.py +14 -0
- data_engine/authoring/builder.py +31 -0
- data_engine/authoring/execution/__init__.py +6 -0
- data_engine/authoring/execution/app.py +6 -0
- data_engine/authoring/execution/context.py +82 -0
- data_engine/authoring/execution/continuous.py +176 -0
- data_engine/authoring/execution/grouped.py +106 -0
- data_engine/authoring/execution/logging.py +83 -0
- data_engine/authoring/execution/polling.py +135 -0
- data_engine/authoring/execution/runner.py +210 -0
- data_engine/authoring/execution/single.py +171 -0
- data_engine/authoring/flow.py +361 -0
- data_engine/authoring/helpers.py +160 -0
- data_engine/authoring/model.py +59 -0
- data_engine/authoring/primitives.py +430 -0
- data_engine/authoring/services.py +42 -0
- data_engine/devtools/__init__.py +3 -0
- data_engine/devtools/project_ast_map.py +503 -0
- data_engine/docs/__init__.py +1 -0
- data_engine/docs/sphinx_source/_static/custom.css +13 -0
- data_engine/docs/sphinx_source/api.rst +42 -0
- data_engine/docs/sphinx_source/conf.py +37 -0
- data_engine/docs/sphinx_source/guides/app-runtime-and-workspaces.md +397 -0
- data_engine/docs/sphinx_source/guides/authoring-flow-modules.md +215 -0
- data_engine/docs/sphinx_source/guides/configuring-flows.md +185 -0
- data_engine/docs/sphinx_source/guides/core-concepts.md +208 -0
- data_engine/docs/sphinx_source/guides/database-methods.md +107 -0
- data_engine/docs/sphinx_source/guides/duckdb-helpers.md +462 -0
- data_engine/docs/sphinx_source/guides/flow-context.md +538 -0
- data_engine/docs/sphinx_source/guides/flow-methods.md +206 -0
- data_engine/docs/sphinx_source/guides/getting-started.md +271 -0
- data_engine/docs/sphinx_source/guides/project-inventory.md +5683 -0
- data_engine/docs/sphinx_source/guides/project-map.md +118 -0
- data_engine/docs/sphinx_source/guides/recipes.md +268 -0
- data_engine/docs/sphinx_source/index.rst +22 -0
- data_engine/domain/__init__.py +92 -0
- data_engine/domain/actions.py +69 -0
- data_engine/domain/catalog.py +128 -0
- data_engine/domain/details.py +214 -0
- data_engine/domain/diagnostics.py +56 -0
- data_engine/domain/errors.py +104 -0
- data_engine/domain/inspection.py +99 -0
- data_engine/domain/logs.py +118 -0
- data_engine/domain/operations.py +172 -0
- data_engine/domain/operator.py +72 -0
- data_engine/domain/runs.py +155 -0
- data_engine/domain/runtime.py +279 -0
- data_engine/domain/source_state.py +17 -0
- data_engine/domain/support.py +54 -0
- data_engine/domain/time.py +23 -0
- data_engine/domain/workspace.py +159 -0
- data_engine/flow_modules/__init__.py +1 -0
- data_engine/flow_modules/flow_module_compiler.py +179 -0
- data_engine/flow_modules/flow_module_loader.py +201 -0
- data_engine/helpers/__init__.py +25 -0
- data_engine/helpers/duckdb.py +705 -0
- data_engine/hosts/__init__.py +1 -0
- data_engine/hosts/daemon/__init__.py +23 -0
- data_engine/hosts/daemon/app.py +221 -0
- data_engine/hosts/daemon/bootstrap.py +69 -0
- data_engine/hosts/daemon/client.py +465 -0
- data_engine/hosts/daemon/commands.py +64 -0
- data_engine/hosts/daemon/composition.py +310 -0
- data_engine/hosts/daemon/constants.py +15 -0
- data_engine/hosts/daemon/entrypoints.py +97 -0
- data_engine/hosts/daemon/lifecycle.py +191 -0
- data_engine/hosts/daemon/manager.py +272 -0
- data_engine/hosts/daemon/ownership.py +126 -0
- data_engine/hosts/daemon/runtime_commands.py +188 -0
- data_engine/hosts/daemon/runtime_control.py +31 -0
- data_engine/hosts/daemon/server.py +84 -0
- data_engine/hosts/daemon/shared_state.py +147 -0
- data_engine/hosts/daemon/state_sync.py +101 -0
- data_engine/platform/__init__.py +1 -0
- data_engine/platform/identity.py +35 -0
- data_engine/platform/local_settings.py +146 -0
- data_engine/platform/theme.py +259 -0
- data_engine/platform/workspace_models.py +190 -0
- data_engine/platform/workspace_policy.py +333 -0
- data_engine/runtime/__init__.py +1 -0
- data_engine/runtime/file_watch.py +185 -0
- data_engine/runtime/ledger_models.py +116 -0
- data_engine/runtime/runtime_db.py +938 -0
- data_engine/runtime/shared_state.py +523 -0
- data_engine/services/__init__.py +49 -0
- data_engine/services/daemon.py +64 -0
- data_engine/services/daemon_state.py +40 -0
- data_engine/services/flow_catalog.py +102 -0
- data_engine/services/flow_execution.py +48 -0
- data_engine/services/ledger.py +85 -0
- data_engine/services/logs.py +65 -0
- data_engine/services/runtime_binding.py +105 -0
- data_engine/services/runtime_execution.py +126 -0
- data_engine/services/runtime_history.py +62 -0
- data_engine/services/settings.py +58 -0
- data_engine/services/shared_state.py +28 -0
- data_engine/services/theme.py +59 -0
- data_engine/services/workspace_provisioning.py +224 -0
- data_engine/services/workspaces.py +74 -0
- data_engine/ui/__init__.py +3 -0
- data_engine/ui/cli/__init__.py +19 -0
- data_engine/ui/cli/app.py +161 -0
- data_engine/ui/cli/commands_doctor.py +178 -0
- data_engine/ui/cli/commands_run.py +80 -0
- data_engine/ui/cli/commands_start.py +100 -0
- data_engine/ui/cli/commands_workspace.py +97 -0
- data_engine/ui/cli/dependencies.py +44 -0
- data_engine/ui/cli/parser.py +56 -0
- data_engine/ui/gui/__init__.py +25 -0
- data_engine/ui/gui/app.py +116 -0
- data_engine/ui/gui/bootstrap.py +487 -0
- data_engine/ui/gui/bootstrapper.py +140 -0
- data_engine/ui/gui/cache_models.py +23 -0
- data_engine/ui/gui/control_support.py +185 -0
- data_engine/ui/gui/controllers/__init__.py +6 -0
- data_engine/ui/gui/controllers/flows.py +439 -0
- data_engine/ui/gui/controllers/runtime.py +245 -0
- data_engine/ui/gui/dialogs/__init__.py +12 -0
- data_engine/ui/gui/dialogs/messages.py +88 -0
- data_engine/ui/gui/dialogs/previews.py +222 -0
- data_engine/ui/gui/helpers/__init__.py +62 -0
- data_engine/ui/gui/helpers/inspection.py +81 -0
- data_engine/ui/gui/helpers/lifecycle.py +112 -0
- data_engine/ui/gui/helpers/scroll.py +28 -0
- data_engine/ui/gui/helpers/theming.py +87 -0
- data_engine/ui/gui/icons/dark_light.svg +12 -0
- data_engine/ui/gui/icons/documentation.svg +1 -0
- data_engine/ui/gui/icons/failed.svg +3 -0
- data_engine/ui/gui/icons/group.svg +4 -0
- data_engine/ui/gui/icons/home.svg +2 -0
- data_engine/ui/gui/icons/manual.svg +2 -0
- data_engine/ui/gui/icons/poll.svg +2 -0
- data_engine/ui/gui/icons/schedule.svg +4 -0
- data_engine/ui/gui/icons/settings.svg +2 -0
- data_engine/ui/gui/icons/started.svg +3 -0
- data_engine/ui/gui/icons/success.svg +3 -0
- data_engine/ui/gui/icons/view-log.svg +3 -0
- data_engine/ui/gui/icons.py +50 -0
- data_engine/ui/gui/launcher.py +48 -0
- data_engine/ui/gui/presenters/__init__.py +72 -0
- data_engine/ui/gui/presenters/docs.py +140 -0
- data_engine/ui/gui/presenters/logs.py +58 -0
- data_engine/ui/gui/presenters/runtime_projection.py +29 -0
- data_engine/ui/gui/presenters/sidebar.py +88 -0
- data_engine/ui/gui/presenters/steps.py +148 -0
- data_engine/ui/gui/presenters/workspace.py +39 -0
- data_engine/ui/gui/presenters/workspace_binding.py +75 -0
- data_engine/ui/gui/presenters/workspace_settings.py +182 -0
- data_engine/ui/gui/preview_models.py +37 -0
- data_engine/ui/gui/render_support.py +241 -0
- data_engine/ui/gui/rendering/__init__.py +12 -0
- data_engine/ui/gui/rendering/artifacts.py +95 -0
- data_engine/ui/gui/rendering/icons.py +50 -0
- data_engine/ui/gui/runtime.py +47 -0
- data_engine/ui/gui/state_support.py +193 -0
- data_engine/ui/gui/support.py +214 -0
- data_engine/ui/gui/surface.py +209 -0
- data_engine/ui/gui/theme.py +720 -0
- data_engine/ui/gui/widgets/__init__.py +34 -0
- data_engine/ui/gui/widgets/config.py +41 -0
- data_engine/ui/gui/widgets/logs.py +62 -0
- data_engine/ui/gui/widgets/panels.py +507 -0
- data_engine/ui/gui/widgets/sidebar.py +130 -0
- data_engine/ui/gui/widgets/steps.py +84 -0
- data_engine/ui/tui/__init__.py +5 -0
- data_engine/ui/tui/app.py +222 -0
- data_engine/ui/tui/bootstrap.py +475 -0
- data_engine/ui/tui/bootstrapper.py +117 -0
- data_engine/ui/tui/controllers/__init__.py +6 -0
- data_engine/ui/tui/controllers/flows.py +349 -0
- data_engine/ui/tui/controllers/runtime.py +167 -0
- data_engine/ui/tui/runtime.py +34 -0
- data_engine/ui/tui/state_support.py +141 -0
- data_engine/ui/tui/support.py +63 -0
- data_engine/ui/tui/theme.py +204 -0
- data_engine/ui/tui/widgets.py +123 -0
- data_engine/views/__init__.py +109 -0
- data_engine/views/actions.py +80 -0
- data_engine/views/artifacts.py +58 -0
- data_engine/views/flow_display.py +69 -0
- data_engine/views/logs.py +54 -0
- data_engine/views/models.py +96 -0
- data_engine/views/presentation.py +133 -0
- data_engine/views/runs.py +62 -0
- data_engine/views/state.py +39 -0
- data_engine/views/status.py +13 -0
- data_engine/views/text.py +109 -0
- py_data_engine-0.1.0.dist-info/METADATA +330 -0
- py_data_engine-0.1.0.dist-info/RECORD +200 -0
- py_data_engine-0.1.0.dist-info/WHEEL +5 -0
- py_data_engine-0.1.0.dist-info/entry_points.txt +2 -0
- py_data_engine-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,449 @@
|
|
|
1
|
+
"""Host-agnostic runtime and daemon-control use cases."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from datetime import UTC, datetime
|
|
7
|
+
from typing import Any
|
|
8
|
+
from collections.abc import Iterable
|
|
9
|
+
|
|
10
|
+
from data_engine.authoring.model import FlowStoppedError
|
|
11
|
+
from data_engine.domain import (
|
|
12
|
+
DaemonLifecyclePolicy,
|
|
13
|
+
DaemonStatusState,
|
|
14
|
+
FlowLogEntry,
|
|
15
|
+
OperationSessionState,
|
|
16
|
+
RuntimeSessionState,
|
|
17
|
+
WorkspaceControlState,
|
|
18
|
+
default_flow_state,
|
|
19
|
+
)
|
|
20
|
+
from data_engine.domain.catalog import FlowCatalogLike
|
|
21
|
+
from data_engine.hosts.daemon.manager import WorkspaceDaemonManager
|
|
22
|
+
from data_engine.platform.workspace_models import WorkspacePaths, authored_workspace_is_available
|
|
23
|
+
from data_engine.services import DaemonService, DaemonStateService, SharedStateService
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass(frozen=True)
|
|
27
|
+
class DaemonCommandResult:
|
|
28
|
+
"""Normalized outcome of one daemon command request."""
|
|
29
|
+
|
|
30
|
+
ok: bool
|
|
31
|
+
error: str = ""
|
|
32
|
+
payload: dict[str, Any] | None = None
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@dataclass(frozen=True)
|
|
36
|
+
class RuntimeSyncState:
|
|
37
|
+
"""Normalized daemon/runtime sync state shared by hosts."""
|
|
38
|
+
|
|
39
|
+
daemon_status: DaemonStatusState
|
|
40
|
+
workspace_control_state: WorkspaceControlState
|
|
41
|
+
runtime_session: RuntimeSessionState
|
|
42
|
+
snapshot_source: str
|
|
43
|
+
snapshot: object
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@dataclass(frozen=True)
|
|
47
|
+
class RuntimeLogMessage:
|
|
48
|
+
"""One host-neutral log line emitted by a runtime completion path."""
|
|
49
|
+
|
|
50
|
+
text: str
|
|
51
|
+
flow_name: str | None = None
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@dataclass(frozen=True)
|
|
55
|
+
class RuntimeSnapshotPresentation:
|
|
56
|
+
"""Normalized runtime snapshot state rebuilt from persisted log history."""
|
|
57
|
+
|
|
58
|
+
operation_tracker: OperationSessionState
|
|
59
|
+
flow_states: dict[str, str]
|
|
60
|
+
|
|
61
|
+
def signature_for(self, runtime_session: RuntimeSessionState) -> tuple[object, ...]:
|
|
62
|
+
"""Return a stable signature for render-diff decisions."""
|
|
63
|
+
return (
|
|
64
|
+
tuple(sorted(self.flow_states.items())),
|
|
65
|
+
tuple(sorted(runtime_session.active_runtime_flow_names)),
|
|
66
|
+
tuple(sorted(runtime_session.active_manual_runs.items())),
|
|
67
|
+
runtime_session.workspace_owned,
|
|
68
|
+
runtime_session.leased_by_machine_id,
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
@dataclass(frozen=True)
|
|
73
|
+
class FlowStateRefreshPlan:
|
|
74
|
+
"""Normalized host-agnostic flow-state refresh plan."""
|
|
75
|
+
|
|
76
|
+
flow_states: dict[str, str]
|
|
77
|
+
changed_flow_names: frozenset[str]
|
|
78
|
+
signature: tuple[object, ...]
|
|
79
|
+
|
|
80
|
+
@property
|
|
81
|
+
def states_changed(self) -> bool:
|
|
82
|
+
"""Return whether any flow-state value changed."""
|
|
83
|
+
return bool(self.changed_flow_names)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
@dataclass(frozen=True)
|
|
87
|
+
class ManualRunCompletion:
|
|
88
|
+
"""Normalized manual-run completion state for operator surfaces."""
|
|
89
|
+
|
|
90
|
+
runtime_session: RuntimeSessionState
|
|
91
|
+
state_updates: dict[str, str]
|
|
92
|
+
log_messages: tuple[RuntimeLogMessage, ...]
|
|
93
|
+
capture_results: bool = False
|
|
94
|
+
normalize_operations: bool = False
|
|
95
|
+
render_durations: bool = False
|
|
96
|
+
show_error_text: str | None = None
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
@dataclass(frozen=True)
|
|
100
|
+
class EngineRunCompletion:
|
|
101
|
+
"""Normalized engine-run completion state for operator surfaces."""
|
|
102
|
+
|
|
103
|
+
runtime_session: RuntimeSessionState
|
|
104
|
+
state_updates: dict[str, str]
|
|
105
|
+
failed_flow_names: tuple[str, ...] = ()
|
|
106
|
+
log_messages: tuple[RuntimeLogMessage, ...] = ()
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
class RuntimeApplication:
|
|
110
|
+
"""Own host-neutral daemon sync and runtime command use cases."""
|
|
111
|
+
|
|
112
|
+
def __init__(
|
|
113
|
+
self,
|
|
114
|
+
*,
|
|
115
|
+
daemon_service: DaemonService,
|
|
116
|
+
daemon_state_service: DaemonStateService,
|
|
117
|
+
shared_state_service: SharedStateService,
|
|
118
|
+
daemon_lifecycle_policy: DaemonLifecyclePolicy = DaemonLifecyclePolicy.EPHEMERAL,
|
|
119
|
+
) -> None:
|
|
120
|
+
self.daemon_service = daemon_service
|
|
121
|
+
self.daemon_state_service = daemon_state_service
|
|
122
|
+
self.shared_state_service = shared_state_service
|
|
123
|
+
self.daemon_lifecycle_policy = daemon_lifecycle_policy
|
|
124
|
+
|
|
125
|
+
def sync_state(
|
|
126
|
+
self,
|
|
127
|
+
*,
|
|
128
|
+
paths: WorkspacePaths,
|
|
129
|
+
daemon_manager: WorkspaceDaemonManager,
|
|
130
|
+
flow_cards,
|
|
131
|
+
runtime_ledger,
|
|
132
|
+
daemon_startup_in_progress: bool = False,
|
|
133
|
+
) -> RuntimeSyncState:
|
|
134
|
+
"""Return normalized daemon/runtime state for one host surface."""
|
|
135
|
+
snapshot = self.daemon_state_service.sync(daemon_manager)
|
|
136
|
+
if snapshot.source == "lease":
|
|
137
|
+
self.shared_state_service.hydrate_local_runtime(paths, runtime_ledger)
|
|
138
|
+
daemon_status = DaemonStatusState.from_snapshot(snapshot)
|
|
139
|
+
return RuntimeSyncState(
|
|
140
|
+
daemon_status=daemon_status,
|
|
141
|
+
workspace_control_state=self.daemon_state_service.control_state(
|
|
142
|
+
daemon_manager,
|
|
143
|
+
snapshot,
|
|
144
|
+
daemon_startup_in_progress=daemon_startup_in_progress,
|
|
145
|
+
),
|
|
146
|
+
runtime_session=RuntimeSessionState.from_daemon_snapshot(snapshot, flow_cards),
|
|
147
|
+
snapshot_source=snapshot.source,
|
|
148
|
+
snapshot=snapshot,
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
def build_runtime_snapshot(
|
|
152
|
+
self,
|
|
153
|
+
*,
|
|
154
|
+
flow_cards: Iterable[FlowCatalogLike],
|
|
155
|
+
log_entries: Iterable[FlowLogEntry],
|
|
156
|
+
runtime_session: RuntimeSessionState,
|
|
157
|
+
now: float,
|
|
158
|
+
) -> RuntimeSnapshotPresentation:
|
|
159
|
+
"""Return normalized runtime flow state rebuilt from catalog metadata and log history."""
|
|
160
|
+
tracker = OperationSessionState.empty()
|
|
161
|
+
states: dict[str, str] = {}
|
|
162
|
+
cards_by_name: dict[str, FlowCatalogLike] = {}
|
|
163
|
+
for card in flow_cards:
|
|
164
|
+
cards_by_name[card.name] = card
|
|
165
|
+
tracker = tracker.ensure_flow(card.name, card.operation_items)
|
|
166
|
+
states[card.name] = card.state if card.valid else "invalid"
|
|
167
|
+
for entry in log_entries:
|
|
168
|
+
event = entry.event
|
|
169
|
+
if event is None or event.flow_name not in cards_by_name:
|
|
170
|
+
continue
|
|
171
|
+
card = cards_by_name[event.flow_name]
|
|
172
|
+
if event.step_name is None:
|
|
173
|
+
if event.status == "failed":
|
|
174
|
+
states[event.flow_name] = "failed"
|
|
175
|
+
elif event.status in {"started", "success", "stopped"}:
|
|
176
|
+
states[event.flow_name] = default_flow_state(card.mode)
|
|
177
|
+
continue
|
|
178
|
+
age_seconds = max((datetime.now(UTC) - entry.created_at_utc).total_seconds(), 0.0)
|
|
179
|
+
tracker, _ = tracker.apply_event(
|
|
180
|
+
event.flow_name,
|
|
181
|
+
card.operation_items,
|
|
182
|
+
event,
|
|
183
|
+
now=now - age_seconds,
|
|
184
|
+
)
|
|
185
|
+
for flow_name in list(states):
|
|
186
|
+
tracker = tracker.normalize_completed(flow_name)
|
|
187
|
+
for flow_name in runtime_session.active_runtime_flow_names:
|
|
188
|
+
card = cards_by_name.get(flow_name)
|
|
189
|
+
if card is None or states.get(flow_name) == "failed":
|
|
190
|
+
continue
|
|
191
|
+
states[flow_name] = (
|
|
192
|
+
"stopping runtime"
|
|
193
|
+
if runtime_session.runtime_stopping
|
|
194
|
+
else ("polling" if card.mode == "poll" else "scheduled")
|
|
195
|
+
)
|
|
196
|
+
for flow_name in runtime_session.active_manual_runs.values():
|
|
197
|
+
if flow_name in states and states.get(flow_name) != "failed":
|
|
198
|
+
states[flow_name] = "running"
|
|
199
|
+
return RuntimeSnapshotPresentation(
|
|
200
|
+
operation_tracker=tracker,
|
|
201
|
+
flow_states=states,
|
|
202
|
+
)
|
|
203
|
+
|
|
204
|
+
def plan_flow_state_refresh(
|
|
205
|
+
self,
|
|
206
|
+
*,
|
|
207
|
+
previous_states: dict[str, str] | None,
|
|
208
|
+
next_states: dict[str, str],
|
|
209
|
+
runtime_session: RuntimeSessionState,
|
|
210
|
+
) -> FlowStateRefreshPlan:
|
|
211
|
+
"""Return one shared diff/signature plan for flow-state refresh decisions."""
|
|
212
|
+
previous = previous_states or {}
|
|
213
|
+
changed_flow_names = frozenset(
|
|
214
|
+
flow_name
|
|
215
|
+
for flow_name in set(previous) | set(next_states)
|
|
216
|
+
if previous.get(flow_name) != next_states.get(flow_name)
|
|
217
|
+
)
|
|
218
|
+
signature = (
|
|
219
|
+
tuple(sorted(next_states.items())),
|
|
220
|
+
tuple(sorted(runtime_session.active_runtime_flow_names)),
|
|
221
|
+
tuple(sorted(runtime_session.active_manual_runs.items())),
|
|
222
|
+
runtime_session.workspace_owned,
|
|
223
|
+
runtime_session.leased_by_machine_id,
|
|
224
|
+
)
|
|
225
|
+
return FlowStateRefreshPlan(
|
|
226
|
+
flow_states=dict(next_states),
|
|
227
|
+
changed_flow_names=changed_flow_names,
|
|
228
|
+
signature=signature,
|
|
229
|
+
)
|
|
230
|
+
|
|
231
|
+
def run_flow(self, paths: WorkspacePaths, *, name: str, wait: bool = False, timeout: float = 2.0) -> DaemonCommandResult:
|
|
232
|
+
"""Request one manual flow run through the daemon."""
|
|
233
|
+
if not authored_workspace_is_available(paths):
|
|
234
|
+
return DaemonCommandResult(ok=False, error="Workspace root is no longer available.")
|
|
235
|
+
return self._spawn_and_request(
|
|
236
|
+
paths,
|
|
237
|
+
{"command": "run_flow", "name": name, "wait": wait},
|
|
238
|
+
timeout=timeout,
|
|
239
|
+
)
|
|
240
|
+
|
|
241
|
+
def start_engine(self, paths: WorkspacePaths, *, timeout: float = 2.0) -> DaemonCommandResult:
|
|
242
|
+
"""Request automated runtime start through the daemon."""
|
|
243
|
+
if not authored_workspace_is_available(paths):
|
|
244
|
+
return DaemonCommandResult(ok=False, error="Workspace root is no longer available.")
|
|
245
|
+
return self._spawn_and_request(paths, {"command": "start_engine"}, timeout=timeout)
|
|
246
|
+
|
|
247
|
+
def refresh_flows(self, paths: WorkspacePaths, *, timeout: float = 5.0) -> DaemonCommandResult:
|
|
248
|
+
"""Request one daemon-side flow refresh through the daemon."""
|
|
249
|
+
return self._spawn_and_request(paths, {"command": "refresh_flows"}, timeout=timeout)
|
|
250
|
+
|
|
251
|
+
def stop_engine(self, paths: WorkspacePaths, *, timeout: float = 2.0) -> DaemonCommandResult:
|
|
252
|
+
"""Request automated runtime stop through the daemon."""
|
|
253
|
+
return self._request(paths, {"command": "stop_engine"}, timeout=timeout)
|
|
254
|
+
|
|
255
|
+
def stop_flow(self, paths: WorkspacePaths, *, name: str, timeout: float = 2.0) -> DaemonCommandResult:
|
|
256
|
+
"""Request one manual flow stop through the daemon."""
|
|
257
|
+
return self._request(paths, {"command": "stop_flow", "name": name}, timeout=timeout)
|
|
258
|
+
|
|
259
|
+
def daemon_status(self, paths: WorkspacePaths, *, timeout: float = 0.0) -> DaemonCommandResult:
|
|
260
|
+
"""Request raw daemon status for host/CLI inspection."""
|
|
261
|
+
return self._request(paths, {"command": "daemon_status"}, timeout=timeout)
|
|
262
|
+
|
|
263
|
+
def shutdown_daemon(self, paths: WorkspacePaths, *, timeout: float = 0.0) -> DaemonCommandResult:
|
|
264
|
+
"""Request daemon shutdown for one workspace."""
|
|
265
|
+
return self._request(paths, {"command": "shutdown_daemon"}, timeout=timeout)
|
|
266
|
+
|
|
267
|
+
def force_shutdown_daemon(self, paths: WorkspacePaths, *, timeout: float = 0.5) -> DaemonCommandResult:
|
|
268
|
+
"""Force-stop the local daemon for one workspace."""
|
|
269
|
+
try:
|
|
270
|
+
self.daemon_service.force_shutdown(paths, timeout=timeout)
|
|
271
|
+
except self.daemon_service.client_error_type as exc:
|
|
272
|
+
return DaemonCommandResult(
|
|
273
|
+
ok=False,
|
|
274
|
+
error=_daemon_command_error_text({"command": "force_shutdown_daemon"}, exc),
|
|
275
|
+
)
|
|
276
|
+
return DaemonCommandResult(ok=True)
|
|
277
|
+
|
|
278
|
+
def spawn_daemon(self, paths: WorkspacePaths) -> DaemonCommandResult:
|
|
279
|
+
"""Start the local daemon using the application lifecycle policy."""
|
|
280
|
+
try:
|
|
281
|
+
self.daemon_service.spawn(
|
|
282
|
+
paths,
|
|
283
|
+
lifecycle_policy=self.daemon_lifecycle_policy,
|
|
284
|
+
)
|
|
285
|
+
except self.daemon_service.client_error_type as exc:
|
|
286
|
+
return DaemonCommandResult(ok=False, error=str(exc))
|
|
287
|
+
return DaemonCommandResult(ok=True)
|
|
288
|
+
|
|
289
|
+
def complete_manual_run(
|
|
290
|
+
self,
|
|
291
|
+
*,
|
|
292
|
+
runtime_session: RuntimeSessionState,
|
|
293
|
+
flow_name: str,
|
|
294
|
+
group_name: str | None,
|
|
295
|
+
flow_mode: str,
|
|
296
|
+
results: object,
|
|
297
|
+
error: object,
|
|
298
|
+
stop_requested: bool,
|
|
299
|
+
) -> ManualRunCompletion:
|
|
300
|
+
"""Return normalized state/log outcomes for one completed manual run."""
|
|
301
|
+
next_runtime = runtime_session.without_manual_group(group_name)
|
|
302
|
+
default_state = _default_state_for_mode(flow_mode)
|
|
303
|
+
if isinstance(error, Exception):
|
|
304
|
+
error_text = _error_text(error)
|
|
305
|
+
if isinstance(error, FlowStoppedError) or stop_requested:
|
|
306
|
+
return ManualRunCompletion(
|
|
307
|
+
runtime_session=next_runtime,
|
|
308
|
+
state_updates={flow_name: default_state},
|
|
309
|
+
log_messages=(RuntimeLogMessage(f"Flow stopped: {flow_name}", flow_name=flow_name),),
|
|
310
|
+
)
|
|
311
|
+
return ManualRunCompletion(
|
|
312
|
+
runtime_session=next_runtime,
|
|
313
|
+
state_updates={flow_name: "failed"},
|
|
314
|
+
log_messages=(RuntimeLogMessage(f"Flow failed: {flow_name}: {error_text}", flow_name=flow_name),),
|
|
315
|
+
show_error_text=f"{flow_name} failed.\n\n{error_text}" if flow_mode == "manual" else None,
|
|
316
|
+
)
|
|
317
|
+
result_count = len(results or [])
|
|
318
|
+
return ManualRunCompletion(
|
|
319
|
+
runtime_session=next_runtime,
|
|
320
|
+
state_updates={flow_name: default_state},
|
|
321
|
+
log_messages=(RuntimeLogMessage(f"Flow finished: {flow_name} with {result_count} result(s)", flow_name=flow_name),),
|
|
322
|
+
capture_results=True,
|
|
323
|
+
normalize_operations=True,
|
|
324
|
+
render_durations=True,
|
|
325
|
+
)
|
|
326
|
+
|
|
327
|
+
def complete_engine_run(
|
|
328
|
+
self,
|
|
329
|
+
*,
|
|
330
|
+
runtime_session: RuntimeSessionState,
|
|
331
|
+
flow_names: tuple[str, ...],
|
|
332
|
+
flow_modes_by_name: dict[str, str],
|
|
333
|
+
error: object,
|
|
334
|
+
runtime_stop_requested: bool,
|
|
335
|
+
flow_stop_requested: bool,
|
|
336
|
+
) -> EngineRunCompletion:
|
|
337
|
+
"""Return normalized state/log outcomes for one completed engine run."""
|
|
338
|
+
next_runtime = runtime_session.with_runtime_flags(active=False, stopping=False).with_active_runtime_flow_names(())
|
|
339
|
+
default_states = {flow_name: _default_state_for_mode(flow_modes_by_name[flow_name]) for flow_name in flow_names}
|
|
340
|
+
if isinstance(error, Exception):
|
|
341
|
+
if isinstance(error, FlowStoppedError) or runtime_stop_requested or flow_stop_requested:
|
|
342
|
+
return EngineRunCompletion(
|
|
343
|
+
runtime_session=next_runtime,
|
|
344
|
+
state_updates=default_states,
|
|
345
|
+
log_messages=tuple(RuntimeLogMessage("Runtime flow stop.", flow_name=flow_name) for flow_name in flow_names),
|
|
346
|
+
)
|
|
347
|
+
return EngineRunCompletion(
|
|
348
|
+
runtime_session=next_runtime,
|
|
349
|
+
state_updates=default_states,
|
|
350
|
+
failed_flow_names=flow_names,
|
|
351
|
+
log_messages=tuple(
|
|
352
|
+
RuntimeLogMessage(f"Runtime failed: {error}", flow_name=flow_name)
|
|
353
|
+
for flow_name in flow_names
|
|
354
|
+
),
|
|
355
|
+
)
|
|
356
|
+
return EngineRunCompletion(
|
|
357
|
+
runtime_session=next_runtime,
|
|
358
|
+
state_updates=default_states,
|
|
359
|
+
log_messages=tuple(RuntimeLogMessage("Runtime flow finish.", flow_name=flow_name) for flow_name in flow_names),
|
|
360
|
+
)
|
|
361
|
+
|
|
362
|
+
def _spawn_and_request(
|
|
363
|
+
self,
|
|
364
|
+
paths: WorkspacePaths,
|
|
365
|
+
payload: dict[str, Any],
|
|
366
|
+
*,
|
|
367
|
+
timeout: float = 0.0,
|
|
368
|
+
) -> DaemonCommandResult:
|
|
369
|
+
spawn_result = self.spawn_daemon(paths)
|
|
370
|
+
if not spawn_result.ok:
|
|
371
|
+
return DaemonCommandResult(
|
|
372
|
+
ok=False,
|
|
373
|
+
error=_daemon_command_error_text(payload, spawn_result.error),
|
|
374
|
+
)
|
|
375
|
+
return self._request(paths, payload, timeout=timeout)
|
|
376
|
+
|
|
377
|
+
def _request(
|
|
378
|
+
self,
|
|
379
|
+
paths: WorkspacePaths,
|
|
380
|
+
payload: dict[str, Any],
|
|
381
|
+
*,
|
|
382
|
+
timeout: float = 0.0,
|
|
383
|
+
) -> DaemonCommandResult:
|
|
384
|
+
try:
|
|
385
|
+
response = self.daemon_service.request(paths, payload, timeout=timeout)
|
|
386
|
+
except self.daemon_service.client_error_type as exc:
|
|
387
|
+
return DaemonCommandResult(ok=False, error=_daemon_command_error_text(payload, exc))
|
|
388
|
+
if not response.get("ok"):
|
|
389
|
+
return DaemonCommandResult(
|
|
390
|
+
ok=False,
|
|
391
|
+
error=_daemon_command_error_text(payload, response.get("error")),
|
|
392
|
+
payload=response,
|
|
393
|
+
)
|
|
394
|
+
return DaemonCommandResult(ok=True, payload=response)
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
def _daemon_command_error_text(payload: dict[str, Any], detail: object | None) -> str:
|
|
398
|
+
"""Return a verbose daemon-command failure message with any available detail."""
|
|
399
|
+
text = str(detail).strip() if detail is not None else ""
|
|
400
|
+
if text:
|
|
401
|
+
return text
|
|
402
|
+
return f"Failed to {_daemon_command_action(payload.get('command'))}. The daemon returned no additional detail."
|
|
403
|
+
|
|
404
|
+
|
|
405
|
+
def _daemon_command_action(command: object) -> str:
|
|
406
|
+
if command == "run_flow":
|
|
407
|
+
return "run the selected flow"
|
|
408
|
+
if command == "start_engine":
|
|
409
|
+
return "start the automated engine"
|
|
410
|
+
if command == "refresh_flows":
|
|
411
|
+
return "refresh flow definitions"
|
|
412
|
+
if command == "stop_engine":
|
|
413
|
+
return "stop the engine"
|
|
414
|
+
if command == "stop_flow":
|
|
415
|
+
return "stop the selected flow"
|
|
416
|
+
if command == "daemon_status":
|
|
417
|
+
return "retrieve daemon status"
|
|
418
|
+
if command == "shutdown_daemon":
|
|
419
|
+
return "shut down the daemon"
|
|
420
|
+
if command == "force_shutdown_daemon":
|
|
421
|
+
return "force-stop the daemon"
|
|
422
|
+
return "complete the requested daemon command"
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
def _default_state_for_mode(mode: str | None) -> str:
|
|
426
|
+
if mode == "poll":
|
|
427
|
+
return "poll ready"
|
|
428
|
+
if mode == "schedule":
|
|
429
|
+
return "schedule ready"
|
|
430
|
+
return "manual"
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
def _error_text(error: Exception) -> str:
|
|
434
|
+
"""Return a non-blank user-facing error detail string."""
|
|
435
|
+
detail = str(error).strip()
|
|
436
|
+
if detail:
|
|
437
|
+
return detail
|
|
438
|
+
return type(error).__name__
|
|
439
|
+
|
|
440
|
+
|
|
441
|
+
__all__ = [
|
|
442
|
+
"DaemonCommandResult",
|
|
443
|
+
"EngineRunCompletion",
|
|
444
|
+
"ManualRunCompletion",
|
|
445
|
+
"RuntimeApplication",
|
|
446
|
+
"RuntimeLogMessage",
|
|
447
|
+
"RuntimeSnapshotPresentation",
|
|
448
|
+
"RuntimeSyncState",
|
|
449
|
+
]
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"""Host-agnostic workspace session use cases."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from data_engine.domain import OperatorSessionState, WorkspaceSessionState
|
|
9
|
+
from data_engine.platform.workspace_models import WorkspacePaths
|
|
10
|
+
from data_engine.services import WorkspaceService
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass(frozen=True)
|
|
14
|
+
class WorkspaceBinding:
|
|
15
|
+
"""Normalized workspace/session binding for one resolved workspace target."""
|
|
16
|
+
|
|
17
|
+
operator_session: OperatorSessionState
|
|
18
|
+
workspace_session: WorkspaceSessionState
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class WorkspaceSessionApplication:
|
|
22
|
+
"""Own workspace discovery and session-state derivation for hosts."""
|
|
23
|
+
|
|
24
|
+
def __init__(self, *, workspace_service: WorkspaceService) -> None:
|
|
25
|
+
self.workspace_service = workspace_service
|
|
26
|
+
|
|
27
|
+
def refresh_session(
|
|
28
|
+
self,
|
|
29
|
+
*,
|
|
30
|
+
workspace_paths: WorkspacePaths,
|
|
31
|
+
override_root: Path | None,
|
|
32
|
+
) -> WorkspaceSessionState:
|
|
33
|
+
"""Return workspace session state rebound to paths plus current discovery."""
|
|
34
|
+
discovered = self.workspace_service.discover(
|
|
35
|
+
app_root=workspace_paths.app_root,
|
|
36
|
+
workspace_collection_root=override_root,
|
|
37
|
+
)
|
|
38
|
+
return WorkspaceSessionState.from_paths(
|
|
39
|
+
workspace_paths,
|
|
40
|
+
override_root=override_root,
|
|
41
|
+
discovered_workspace_ids=(item.workspace_id for item in discovered),
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
def bind_workspace(
|
|
45
|
+
self,
|
|
46
|
+
*,
|
|
47
|
+
workspace_paths: WorkspacePaths,
|
|
48
|
+
override_root: Path | None,
|
|
49
|
+
) -> WorkspaceBinding:
|
|
50
|
+
"""Return a fresh operator/session binding for one resolved workspace target."""
|
|
51
|
+
workspace_session = self.refresh_session(
|
|
52
|
+
workspace_paths=workspace_paths,
|
|
53
|
+
override_root=override_root,
|
|
54
|
+
)
|
|
55
|
+
operator_session = OperatorSessionState.from_paths(
|
|
56
|
+
workspace_paths,
|
|
57
|
+
override_root=override_root,
|
|
58
|
+
).with_workspace(workspace_session)
|
|
59
|
+
return WorkspaceBinding(
|
|
60
|
+
operator_session=operator_session,
|
|
61
|
+
workspace_session=workspace_session,
|
|
62
|
+
)
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"""Authoring DSL and core flow model primitives."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from importlib import import_module
|
|
6
|
+
|
|
7
|
+
__all__ = ["Batch", "FileRef", "Flow", "FlowContext", "discover_flows", "load_flow", "run"]
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def __getattr__(name: str):
|
|
11
|
+
if name not in __all__:
|
|
12
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
13
|
+
builder = import_module("data_engine.authoring.builder")
|
|
14
|
+
return getattr(builder, name)
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""Public facade for the authoring DSL and runtime entrypoints."""
|
|
2
|
+
|
|
3
|
+
from data_engine.authoring.execution import _FlowRuntime
|
|
4
|
+
from data_engine.authoring.execution import _GroupedFlowRuntime
|
|
5
|
+
from data_engine.authoring.flow import Flow
|
|
6
|
+
from data_engine.authoring.flow import discover_flows
|
|
7
|
+
from data_engine.authoring.flow import load_flow
|
|
8
|
+
from data_engine.authoring.flow import run
|
|
9
|
+
from data_engine.authoring.helpers import _title_case_words
|
|
10
|
+
from data_engine.authoring.primitives import Batch
|
|
11
|
+
from data_engine.authoring.primitives import FileRef
|
|
12
|
+
from data_engine.authoring.primitives import FlowContext
|
|
13
|
+
from data_engine.authoring.primitives import MirrorContext
|
|
14
|
+
from data_engine.authoring.primitives import SourceContext
|
|
15
|
+
from data_engine.authoring.primitives import collect_files
|
|
16
|
+
|
|
17
|
+
__all__ = [
|
|
18
|
+
"Batch",
|
|
19
|
+
"FileRef",
|
|
20
|
+
"Flow",
|
|
21
|
+
"FlowContext",
|
|
22
|
+
"MirrorContext",
|
|
23
|
+
"SourceContext",
|
|
24
|
+
"_FlowRuntime",
|
|
25
|
+
"_GroupedFlowRuntime",
|
|
26
|
+
"_title_case_words",
|
|
27
|
+
"collect_files",
|
|
28
|
+
"discover_flows",
|
|
29
|
+
"load_flow",
|
|
30
|
+
"run",
|
|
31
|
+
]
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
"""Runtime context building helpers for authored flows."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
import hashlib
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from uuid import uuid4
|
|
9
|
+
|
|
10
|
+
from data_engine.authoring.primitives import FlowContext, MirrorContext, SourceContext, WatchSpec, WorkspaceConfigContext
|
|
11
|
+
from data_engine.domain.source_state import SourceSignature
|
|
12
|
+
from data_engine.domain.time import utcnow_text
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass(frozen=True)
|
|
16
|
+
class _QueuedJob:
|
|
17
|
+
"""One queued runtime job for a flow and an optional concrete source file."""
|
|
18
|
+
|
|
19
|
+
flow: "Flow"
|
|
20
|
+
source_path: Path | None
|
|
21
|
+
batch_signatures: tuple[SourceSignature, ...] = ()
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class RuntimeContextBuilder:
|
|
25
|
+
"""Build runtime flow contexts for concrete or root-level executions."""
|
|
26
|
+
|
|
27
|
+
@staticmethod
|
|
28
|
+
def _source_key_text(*, source_path: Path | None, relative_path: Path | None) -> str | None:
|
|
29
|
+
if relative_path is not None:
|
|
30
|
+
return relative_path.as_posix()
|
|
31
|
+
if source_path is not None:
|
|
32
|
+
return source_path.as_posix()
|
|
33
|
+
return None
|
|
34
|
+
|
|
35
|
+
@classmethod
|
|
36
|
+
def _source_file_hash(cls, *, source_path: Path | None, relative_path: Path | None) -> str | None:
|
|
37
|
+
source_key = cls._source_key_text(source_path=source_path, relative_path=relative_path)
|
|
38
|
+
if source_key is None:
|
|
39
|
+
return None
|
|
40
|
+
return hashlib.sha1(source_key.encode("utf-8")).hexdigest()
|
|
41
|
+
|
|
42
|
+
def new_run_id(self) -> str:
|
|
43
|
+
return uuid4().hex
|
|
44
|
+
|
|
45
|
+
def build(self, flow: "Flow", source_path: Path | None, *, run_id: str) -> FlowContext:
|
|
46
|
+
metadata: dict[str, object] = {"started_at_utc": utcnow_text(), "run_id": run_id, "step_outputs": {}}
|
|
47
|
+
context = FlowContext(
|
|
48
|
+
flow_name=flow.name,
|
|
49
|
+
group=flow.group,
|
|
50
|
+
metadata=metadata,
|
|
51
|
+
config=WorkspaceConfigContext(workspace_root=flow._workspace_root),
|
|
52
|
+
)
|
|
53
|
+
source_root: Path | None = None
|
|
54
|
+
resolved_source_path: Path | None = None
|
|
55
|
+
relative_path: Path | None = None
|
|
56
|
+
trigger = flow.trigger
|
|
57
|
+
if isinstance(trigger, WatchSpec) and trigger.source is not None:
|
|
58
|
+
if trigger.source.exists() and trigger.source.is_dir():
|
|
59
|
+
source_root = trigger.source
|
|
60
|
+
if source_path is not None:
|
|
61
|
+
resolved_source_path = source_path
|
|
62
|
+
relative_path = source_path.relative_to(trigger.source)
|
|
63
|
+
elif trigger.source.exists() and trigger.source.is_file():
|
|
64
|
+
resolved_source = source_path if source_path is not None else trigger.source
|
|
65
|
+
resolved_source_path = resolved_source
|
|
66
|
+
source_root = trigger.source.parent
|
|
67
|
+
relative_path = Path(resolved_source.name)
|
|
68
|
+
if source_root is not None:
|
|
69
|
+
context.source = SourceContext(root=source_root, path=resolved_source_path, relative_path=relative_path)
|
|
70
|
+
file_hash = self._source_file_hash(source_path=resolved_source_path, relative_path=relative_path)
|
|
71
|
+
if file_hash is not None:
|
|
72
|
+
context.metadata["file_hash"] = file_hash
|
|
73
|
+
if flow.mirror_spec is not None:
|
|
74
|
+
context.mirror = MirrorContext(
|
|
75
|
+
root=flow.mirror_spec.root,
|
|
76
|
+
source_path=context.source.path if context.source is not None else None,
|
|
77
|
+
relative_path=context.source.relative_path if context.source is not None else None,
|
|
78
|
+
)
|
|
79
|
+
return context
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
__all__ = ["RuntimeContextBuilder", "_QueuedJob"]
|