interlaced 2.1.0__tar.gz → 2.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {interlaced-2.1.0/src/interlaced.egg-info → interlaced-2.2.0}/PKG-INFO +1 -1
- {interlaced-2.1.0 → interlaced-2.2.0}/pyproject.toml +1 -1
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/cli/main.py +2 -1
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/dsl/decorators.py +1 -1
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/plan/apply.py +24 -1
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/plan/differ.py +18 -3
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/plan/plan.py +19 -2
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/service/app.py +124 -18
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/service/ui/app.css +39 -0
- interlaced-2.2.0/src/interlace/service/ui/favicon.svg +28 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/service/ui/index.html +3 -3
- interlaced-2.2.0/src/interlace/service/ui/js/timeline.js +157 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/service/ui/js/ui.js +4 -3
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/service/ui/js/views/overview.js +34 -15
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/service/ui/js/views/query.js +8 -1
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/service/ui/js/views/runs.js +116 -83
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/state/janitor.py +17 -3
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/state/store.py +53 -13
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/strategies/__init__.py +6 -0
- interlaced-2.2.0/src/interlace/strategies/hash_merge.py +119 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/templates/events/template.yaml +2 -1
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/templates/github/template.yaml +2 -1
- interlaced-2.2.0/src/interlace/templates/postgres/template.yaml +4 -0
- {interlaced-2.1.0 → interlaced-2.2.0/src/interlaced.egg-info}/PKG-INFO +1 -1
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlaced.egg-info/SOURCES.txt +2 -0
- interlaced-2.1.0/src/interlace/service/ui/favicon.svg +0 -1
- interlaced-2.1.0/src/interlace/templates/postgres/template.yaml +0 -3
- {interlaced-2.1.0 → interlaced-2.2.0}/LICENSE +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/MANIFEST.in +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/README.md +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/setup.cfg +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/__init__.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/checks/__init__.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/checks/builtin.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/checks/runner.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/checks/spec.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/cli/__init__.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/config/__init__.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/config/config.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/contracts.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/dsl/__init__.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/dsl/discovery.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/dsl/sql_config.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/engines/__init__.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/engines/adbc.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/engines/base.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/engines/bigquery.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/engines/duckdb.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/engines/postgres.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/engines/quack.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/engines/redshift.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/engines/registry.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/engines/snowflake.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/engines/spark.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/exceptions.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/graph/__init__.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/graph/column_lineage.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/graph/dag.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/graph/project.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/graph/selectors.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/ir/__init__.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/ir/canonicalize.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/ir/fingerprint.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/ir/relation.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/plan/__init__.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/plan/resolve.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/plan/run.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/project.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/py.typed +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/query.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/runtime/__init__.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/runtime/handles.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/runtime/python_model.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/scaffold.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/scheduler/__init__.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/scheduler/engine.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/scheduler/triggers.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/scheduler/worker.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/service/__init__.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/service/auth.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/service/ui/js/api.js +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/service/ui/js/app.js +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/service/ui/js/dag.js +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/service/ui/js/views/checks.js +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/service/ui/js/views/environments.js +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/service/ui/js/views/lineage.js +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/service/ui/js/views/models.js +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/service/ui/js/views/plan.js +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/service/ui/js/views/streams.js +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/service/ui/js/views/system.js +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/sinks.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/sources/__init__.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/sources/auth.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/sources/rest.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/state/__init__.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/state/interval.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/state/snapshot.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/strategies/append.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/strategies/base.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/strategies/full_merge.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/strategies/incremental_by_time.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/strategies/merge.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/strategies/replace.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/strategies/replace_in_place.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/strategies/scd.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/strategies/view.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/streaming/__init__.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/streaming/log.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/streaming/materializer.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/streaming/schema.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/templates/events/README.md +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/templates/events/generate.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/templates/events/interlace.yaml +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/templates/events/models/events.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/templates/events/models/events_by_minute.sql +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/templates/events/models/events_by_type.sql +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/templates/events/models/top_users.sql +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/templates/events/models/user_spend.sql +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/templates/github/README.md +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/templates/github/interlace.yaml +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/templates/github/models/github_issues.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/templates/github/models/issues_by_state.sql +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/templates/postgres/README.md +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/templates/postgres/docker-compose.yml +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/templates/postgres/init/seed.sql +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/templates/postgres/interlace.yaml +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/templates/postgres/models/orders.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/templates/postgres/models/orders_by_status.sql +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/templates/quickstart/README.md +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/templates/quickstart/interlace.yaml +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/templates/quickstart/models/enriched_events.py +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/templates/quickstart/models/event_summary.sql +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/templates/quickstart/models/raw_events.sql +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlace/templates/quickstart/template.yaml +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlaced.egg-info/dependency_links.txt +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlaced.egg-info/entry_points.txt +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlaced.egg-info/requires.txt +0 -0
- {interlaced-2.1.0 → interlaced-2.2.0}/src/interlaced.egg-info/top_level.txt +0 -0
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "interlaced"
|
|
7
|
-
version = "2.
|
|
7
|
+
version = "2.2.0"
|
|
8
8
|
description = "Python/SQL-first data platform: transformation, built-in orchestration, and durable streaming ingestion"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.12"
|
|
@@ -252,7 +252,8 @@ def init(
|
|
|
252
252
|
table.add_column("Description", style="dim")
|
|
253
253
|
table.add_column("Needs", style="dim")
|
|
254
254
|
for info in list_templates():
|
|
255
|
-
|
|
255
|
+
# escape: a description may contain [sources]-style brackets Rich would eat as markup
|
|
256
|
+
table.add_row(info.name, escape(info.description), ", ".join(info.requires_env) or "—")
|
|
256
257
|
console.print(table)
|
|
257
258
|
return
|
|
258
259
|
try:
|
|
@@ -22,7 +22,7 @@ ModelFn = Callable[..., Any]
|
|
|
22
22
|
# (a fingerprinted snapshot read through an environment view); `table`/`file` are
|
|
23
23
|
# terminal deliveries into a destination interlace does not own.
|
|
24
24
|
_MATERIALISATIONS = frozenset({"virtual", "view", "ephemeral", "table", "file"})
|
|
25
|
-
_KEYED_STRATEGIES = frozenset({"merge", "full_merge", "scd"})
|
|
25
|
+
_KEYED_STRATEGIES = frozenset({"merge", "full_merge", "hash_merge", "scd"})
|
|
26
26
|
_DRIFT_MODES = frozenset({"evolve", "reject", "quarantine"})
|
|
27
27
|
|
|
28
28
|
|
|
@@ -38,6 +38,7 @@ from interlace.state.interval import Interval
|
|
|
38
38
|
from interlace.state.store import StateStore
|
|
39
39
|
from interlace.strategies import Strategy, resolve_strategy
|
|
40
40
|
from interlace.strategies.base import RowCounts
|
|
41
|
+
from interlace.strategies.hash_merge import HashMerge
|
|
41
42
|
|
|
42
43
|
|
|
43
44
|
@dataclass
|
|
@@ -107,6 +108,10 @@ async def _merge_python_output(
|
|
|
107
108
|
pre_statements, source, columns = await _align_stage_to_target(
|
|
108
109
|
engine, stage, target, exclude=strategy.managed_columns
|
|
109
110
|
)
|
|
111
|
+
elif isinstance(strategy, HashMerge):
|
|
112
|
+
# hash_merge builds its _hash from the column list; on the first build the target
|
|
113
|
+
# doesn't exist yet (so no align pass ran), so take the columns from the staged output
|
|
114
|
+
columns = [c for c in await engine.describe(stage) if c not in strategy.managed_columns]
|
|
110
115
|
|
|
111
116
|
relation = SqlRelation(ast=source)
|
|
112
117
|
statements = strategy.plan_statements(relation, target, engine.caps, None, columns)
|
|
@@ -409,6 +414,21 @@ async def _run_backfill(
|
|
|
409
414
|
target_engine = registry.require(model.engine, model=model.name)
|
|
410
415
|
resolution = await _stage_cross_engine_inputs(model, compiled, registry, physical, staged, stage_lock, result)
|
|
411
416
|
|
|
417
|
+
if task.reuse_existing and await target_engine.table_exists(snapshot.physical_table):
|
|
418
|
+
# fingerprint already materialised (e.g. by another environment): the content-
|
|
419
|
+
# addressed table exists, so skip the (re)build compute — record the snapshot for
|
|
420
|
+
# this env's promotion, gate on checks against the existing table, and let the
|
|
421
|
+
# caller swap this environment's view onto the shared table. The table_exists guard
|
|
422
|
+
# is what makes the differ's optimistic reuse safe: a snapshot row can outlive its
|
|
423
|
+
# table (an in-memory warehouse across processes, a gc'd table) — then we fall
|
|
424
|
+
# through and build for real.
|
|
425
|
+
await state.add_snapshot(snapshot) # idempotent — the row may already exist
|
|
426
|
+
if snapshot.name not in result.built and snapshot.name not in result.reused:
|
|
427
|
+
result.reused.append(snapshot.name)
|
|
428
|
+
await _gate_checks(model, compiled, target_engine, state, plan.environment, result, resolution)
|
|
429
|
+
result.timings[snapshot.name] = result.timings.get(snapshot.name, 0.0) + (time.perf_counter() - task_started)
|
|
430
|
+
return
|
|
431
|
+
|
|
412
432
|
if model.ast is None: # Python model: run the function, load Arrow into the snapshot table
|
|
413
433
|
if model.materialise != "virtual":
|
|
414
434
|
raise PlanError(
|
|
@@ -669,7 +689,10 @@ async def apply(
|
|
|
669
689
|
|
|
670
690
|
mapping = {name: compiled.models[name].fingerprint for name in plan.promote}
|
|
671
691
|
await state.promote(plan.environment, mapping)
|
|
672
|
-
|
|
692
|
+
# ephemeral models are tracked in the mapping (so re-plans stay clean) but are inlined
|
|
693
|
+
# into consumers — they have no promotable table/view, so the user-facing count omits
|
|
694
|
+
# them, keeping "promoted N" consistent with the N build rows shown
|
|
695
|
+
result.promoted = sum(1 for name in mapping if compiled.models[name].materialise != "ephemeral")
|
|
673
696
|
|
|
674
697
|
# deleted models: drop their env view and demote them, or the view serves the
|
|
675
698
|
# last snapshot forever and pins it against gc
|
|
@@ -243,7 +243,7 @@ def _schedule_reuse(plan: Plan, model: CompiledModel, previous: Snapshot, enviro
|
|
|
243
243
|
)
|
|
244
244
|
|
|
245
245
|
|
|
246
|
-
_HISTORY_STRATEGIES = frozenset({"merge", "full_merge", "scd", "incremental_by_time"})
|
|
246
|
+
_HISTORY_STRATEGIES = frozenset({"merge", "full_merge", "hash_merge", "scd", "incremental_by_time"})
|
|
247
247
|
"""Strategies whose targets accumulate state a rebuild would destroy."""
|
|
248
248
|
|
|
249
249
|
|
|
@@ -303,6 +303,13 @@ async def diff(
|
|
|
303
303
|
for name, fingerprint in current.items()
|
|
304
304
|
if name in compiled.models and fingerprint != compiled.models[name].fingerprint
|
|
305
305
|
)
|
|
306
|
+
# Fingerprints already materialised (by any prior apply — most usefully another
|
|
307
|
+
# environment): building these again would recompute an identical, content-addressed
|
|
308
|
+
# table, so schedule a reuse (record + view-swap) instead of a rebuild.
|
|
309
|
+
already_built = await state.get_snapshots((name, compiled.models[name].fingerprint) for name in selected)
|
|
310
|
+
|
|
311
|
+
def is_materialised(model: CompiledModel) -> bool:
|
|
312
|
+
return (model.name, model.fingerprint) in already_built
|
|
306
313
|
|
|
307
314
|
for model in compiled.ordered(): # topo order: upstream impact known before downstream
|
|
308
315
|
previous_fingerprint = current.get(model.name)
|
|
@@ -311,7 +318,13 @@ async def diff(
|
|
|
311
318
|
impact[model.name] = "semantic"
|
|
312
319
|
if model.name in selected:
|
|
313
320
|
plan.changes.append(ModelChange(model.name, ChangeType.ADDED, None, None, model.fingerprint))
|
|
314
|
-
schedule_build(
|
|
321
|
+
schedule_build(
|
|
322
|
+
plan,
|
|
323
|
+
model,
|
|
324
|
+
snapshot_of(model, ChangeCategory.BREAKING),
|
|
325
|
+
environment,
|
|
326
|
+
reuse_existing=is_materialised(model),
|
|
327
|
+
)
|
|
315
328
|
continue
|
|
316
329
|
|
|
317
330
|
if previous_fingerprint == model.fingerprint:
|
|
@@ -373,7 +386,9 @@ async def diff(
|
|
|
373
386
|
)
|
|
374
387
|
schedule_build(plan, model, snapshot, environment, seed_from=previous.physical_table) # type: ignore[union-attr]
|
|
375
388
|
elif rebuild:
|
|
376
|
-
schedule_build(
|
|
389
|
+
schedule_build(
|
|
390
|
+
plan, model, snapshot_of(model, category), environment, reuse_existing=is_materialised(model)
|
|
391
|
+
)
|
|
377
392
|
else:
|
|
378
393
|
_schedule_reuse(plan, model, previous, environment) # type: ignore[arg-type] # previous is not None here
|
|
379
394
|
|
|
@@ -55,6 +55,10 @@ class BackfillTask:
|
|
|
55
55
|
# the strategy runs — history moves to the new fingerprint, the old table stays
|
|
56
56
|
# as the rollback until gc.
|
|
57
57
|
seed_from: TableRef | None = None
|
|
58
|
+
# This exact fingerprint is already materialised (a prior apply, often in another
|
|
59
|
+
# environment): the physical table exists and its content is fingerprint-pinned, so
|
|
60
|
+
# apply skips the build compute and only gates checks + swaps this env's view.
|
|
61
|
+
reuse_existing: bool = False
|
|
58
62
|
|
|
59
63
|
|
|
60
64
|
@dataclass(frozen=True)
|
|
@@ -121,7 +125,13 @@ def env_view(environment: str, model_name: str) -> TableRef:
|
|
|
121
125
|
|
|
122
126
|
|
|
123
127
|
def schedule_build(
|
|
124
|
-
plan: Plan,
|
|
128
|
+
plan: Plan,
|
|
129
|
+
model: CompiledModel,
|
|
130
|
+
snapshot: Snapshot,
|
|
131
|
+
environment: str,
|
|
132
|
+
*,
|
|
133
|
+
seed_from: TableRef | None = None,
|
|
134
|
+
reuse_existing: bool = False,
|
|
125
135
|
) -> None:
|
|
126
136
|
"""Add the right tasks for a model: ephemeral builds nothing; a terminal
|
|
127
137
|
table/file builds (delivers) but gets no environment view; a virtual/view model
|
|
@@ -130,7 +140,14 @@ def schedule_build(
|
|
|
130
140
|
An incremental_by_time model (virtual, or a terminal ``table``) cannot build
|
|
131
141
|
without a window, so an apply fills the latest grain interval — the same default
|
|
132
142
|
as ``interlace run`` — leaving history to ``run --start/--end``.
|
|
143
|
+
|
|
144
|
+
``reuse_existing`` (the fingerprint is already materialised) skips the compute for a
|
|
145
|
+
plain virtual/view build — never for a terminal delivery, a forward-only seed, or an
|
|
146
|
+
incremental window, which must always run.
|
|
133
147
|
"""
|
|
148
|
+
# only a plain (non-seeded, non-windowed) virtual/view build can skip its compute:
|
|
149
|
+
# a terminal always delivers, a seed must copy history, an interval must fill
|
|
150
|
+
reuse = reuse_existing and seed_from is None and model.materialise in ("virtual", "view")
|
|
134
151
|
if model.materialise == "ephemeral": # inlined into consumers, never built
|
|
135
152
|
return
|
|
136
153
|
wants_view = model.materialise in ("virtual", "view") # terminal table/file has no env view
|
|
@@ -164,7 +181,7 @@ def schedule_build(
|
|
|
164
181
|
plan.backfills.append(BackfillTask(snapshot=snapshot, interval=window, seed_from=seed_from))
|
|
165
182
|
add_view()
|
|
166
183
|
return
|
|
167
|
-
plan.backfills.append(BackfillTask(snapshot=snapshot, seed_from=seed_from))
|
|
184
|
+
plan.backfills.append(BackfillTask(snapshot=snapshot, seed_from=seed_from, reuse_existing=reuse))
|
|
168
185
|
add_view()
|
|
169
186
|
|
|
170
187
|
|
|
@@ -4,9 +4,10 @@ A read + trigger surface over a project: list/inspect models, preview a plan
|
|
|
4
4
|
(with per-change SQL + impacted columns), apply it (build changed snapshots and
|
|
5
5
|
promote — guarded against breaking changes unless forced), inspect environments,
|
|
6
6
|
list/inspect runs, and enqueue runs onto the durable queue (a running ``interlace
|
|
7
|
-
scheduler`` drains them). The project is
|
|
8
|
-
|
|
9
|
-
the
|
|
7
|
+
scheduler`` drains them). The project is compiled at startup and recompiled on
|
|
8
|
+
demand when a model file changes on disk (see ``reload_if_stale``), so a Plan/Apply
|
|
9
|
+
from the UI reflects live edits — matching what ``interlace plan`` shows; the
|
|
10
|
+
warehouse engine and control-plane store are opened for the app's lifetime. msgspec structs are the wire types (Litestar serialises them
|
|
10
11
|
natively). Scoped API-key auth is enforced once a key exists (see auth.py), and
|
|
11
12
|
OpenAPI docs render via Scalar at ``/schema/scalar``.
|
|
12
13
|
"""
|
|
@@ -57,6 +58,52 @@ from interlace.streaming.materializer import (
|
|
|
57
58
|
)
|
|
58
59
|
from interlace.streaming.schema import partition_rows, validate_rows, validate_rows_evolve
|
|
59
60
|
|
|
61
|
+
# ---- live reload ---------------------------------------------------------------
|
|
62
|
+
# The project is compiled once at startup, then recompiled on demand when a model
|
|
63
|
+
# file changes on disk — so editing a `.sql`/`.py` model and pressing Plan/Apply in
|
|
64
|
+
# the UI reflects the edit, matching what `interlace plan` (a fresh process) shows.
|
|
65
|
+
# Only the model graph is re-derived; changing engine/stream/path topology in
|
|
66
|
+
# interlace.yaml still needs a daemon restart.
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _source_mtime(root: Path, model_paths: list[str]) -> float:
|
|
70
|
+
"""Newest mtime across the config file and every model source. Cheap staleness
|
|
71
|
+
probe (a stat walk); recompile only happens when this advances."""
|
|
72
|
+
from interlace.config.config import CONFIG_FILE
|
|
73
|
+
|
|
74
|
+
latest = 0.0
|
|
75
|
+
config = root / CONFIG_FILE
|
|
76
|
+
if config.exists():
|
|
77
|
+
latest = config.stat().st_mtime
|
|
78
|
+
for relative in model_paths:
|
|
79
|
+
base = root / relative
|
|
80
|
+
if not base.is_dir():
|
|
81
|
+
continue
|
|
82
|
+
for file in base.rglob("*"):
|
|
83
|
+
if file.suffix in (".sql", ".py") and file.is_file():
|
|
84
|
+
latest = max(latest, file.stat().st_mtime)
|
|
85
|
+
return latest
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
async def reload_if_stale(state: State) -> None:
|
|
89
|
+
"""Recompile from disk if any model source changed since the last compile.
|
|
90
|
+
|
|
91
|
+
Serialised by a lock so concurrent requests recompile at most once, and a
|
|
92
|
+
no-op (one stat walk) when nothing changed. Refreshes only the compile-derived
|
|
93
|
+
state — the graph, whole-project lineage, stream→consumer map — and clears the
|
|
94
|
+
describe cache (new fingerprints mean new tables to re-describe)."""
|
|
95
|
+
async with state.reload_lock:
|
|
96
|
+
mtime = await asyncio.to_thread(_source_mtime, state.root, state.model_paths)
|
|
97
|
+
if mtime <= state.source_mtime:
|
|
98
|
+
return
|
|
99
|
+
project = await asyncio.to_thread(Project.load, state.root)
|
|
100
|
+
compiled = await asyncio.to_thread(project.compile)
|
|
101
|
+
state.compiled = compiled
|
|
102
|
+
state.lineage = column_lineage(compiled)
|
|
103
|
+
state.stream_consumer_map = {name: sorted(stream_consumers(compiled, name)) for name in state.streams}
|
|
104
|
+
state.describe_cache = {}
|
|
105
|
+
state.source_mtime = mtime
|
|
106
|
+
|
|
60
107
|
|
|
61
108
|
class ModelInfo(msgspec.Struct):
|
|
62
109
|
name: str
|
|
@@ -71,6 +118,7 @@ class ModelInfo(msgspec.Struct):
|
|
|
71
118
|
schedule: dict[str, str] | None
|
|
72
119
|
engine: str = "default"
|
|
73
120
|
language: str = "sql" # "sql" | "python"
|
|
121
|
+
has_checks: bool = False # declares SQL or Python checks — the runs view flags per-model check status
|
|
74
122
|
|
|
75
123
|
|
|
76
124
|
class ModelDetail(msgspec.Struct):
|
|
@@ -123,6 +171,8 @@ class RunInfo(msgspec.Struct):
|
|
|
123
171
|
# how the run came to be — the enqueue key's prefix names the trigger
|
|
124
172
|
# (cron: / interval: / api: / stream:)
|
|
125
173
|
idempotency_key: str | None = None
|
|
174
|
+
environment: str | None = None # the env it built into (once it has succeeded)
|
|
175
|
+
duration: float | None = None # wall-clock seconds, run.started → terminal
|
|
126
176
|
|
|
127
177
|
|
|
128
178
|
class CreateRun(msgspec.Struct):
|
|
@@ -184,6 +234,8 @@ class ApplyResponse(msgspec.Struct):
|
|
|
184
234
|
# per-model row movement (inserted/updated/deleted) and build seconds
|
|
185
235
|
rows: dict[str, dict[str, int]] = msgspec.field(default_factory=dict)
|
|
186
236
|
timings: dict[str, float] = msgspec.field(default_factory=dict)
|
|
237
|
+
gated: list[str] = msgspec.field(default_factory=list) # terminals recorded but not delivered (env gate)
|
|
238
|
+
checks: list[CheckOutcomeInfo] = msgspec.field(default_factory=list) # so a UI apply can show check results
|
|
187
239
|
|
|
188
240
|
|
|
189
241
|
class CheckResultInfo(msgspec.Struct):
|
|
@@ -371,7 +423,7 @@ def _output(model: CompiledModel) -> str:
|
|
|
371
423
|
return model.materialise
|
|
372
424
|
|
|
373
425
|
|
|
374
|
-
def _info(model: CompiledModel) -> ModelInfo:
|
|
426
|
+
def _info(model: CompiledModel, has_checks: bool = False) -> ModelInfo:
|
|
375
427
|
return ModelInfo(
|
|
376
428
|
name=model.name,
|
|
377
429
|
output=_output(model),
|
|
@@ -385,6 +437,7 @@ def _info(model: CompiledModel) -> ModelInfo:
|
|
|
385
437
|
schedule=model.schedule,
|
|
386
438
|
engine=model.engine,
|
|
387
439
|
language="python" if model.ast is None else "sql",
|
|
440
|
+
has_checks=has_checks,
|
|
388
441
|
)
|
|
389
442
|
|
|
390
443
|
|
|
@@ -400,12 +453,17 @@ async def ui_redirect() -> Redirect:
|
|
|
400
453
|
|
|
401
454
|
@get("/models")
|
|
402
455
|
async def get_models(state: State) -> list[ModelInfo]:
|
|
456
|
+
await reload_if_stale(state)
|
|
403
457
|
compiled: CompiledProject = state.compiled
|
|
404
|
-
return [
|
|
458
|
+
return [
|
|
459
|
+
_info(compiled.models[name], bool(compiled.models[name].checks) or bool(compiled.python_checks.get(name)))
|
|
460
|
+
for name in compiled.graph.topological_sort()
|
|
461
|
+
]
|
|
405
462
|
|
|
406
463
|
|
|
407
464
|
@get("/models/{name:str}")
|
|
408
465
|
async def get_model(name: FromPath[str], state: State) -> ModelDetail:
|
|
466
|
+
await reload_if_stale(state)
|
|
409
467
|
compiled: CompiledProject = state.compiled
|
|
410
468
|
if name not in compiled.models:
|
|
411
469
|
raise NotFoundException(detail=f"unknown model: {name}")
|
|
@@ -437,6 +495,7 @@ async def get_model_impact(name: FromPath[str], state: State, column: FromQuery[
|
|
|
437
495
|
transitively derived from it, plus opaque consumers (Python / ``*`` models)."""
|
|
438
496
|
from interlace.graph.column_lineage import column_impact
|
|
439
497
|
|
|
498
|
+
await reload_if_stale(state)
|
|
440
499
|
compiled: CompiledProject = state.compiled
|
|
441
500
|
if name not in compiled.models:
|
|
442
501
|
raise NotFoundException(detail=f"unknown model: {name}")
|
|
@@ -458,6 +517,7 @@ async def get_plan(
|
|
|
458
517
|
"""Preview a plan. ``select`` takes comma-separated selectors (same grammar as
|
|
459
518
|
POST /apply's ``selectors``); ``forward_only`` previews history-inheriting plans
|
|
460
519
|
— so what you preview is exactly what POST /apply will do."""
|
|
520
|
+
await reload_if_stale(state)
|
|
461
521
|
env = environment or state.environment
|
|
462
522
|
compiled: CompiledProject = state.compiled
|
|
463
523
|
selectors = [part.strip() for part in select.split(",") if part.strip()] if select else []
|
|
@@ -500,6 +560,7 @@ async def get_plan(
|
|
|
500
560
|
|
|
501
561
|
@get("/environments")
|
|
502
562
|
async def get_environments(state: State) -> list[EnvironmentInfo]:
|
|
563
|
+
await reload_if_stale(state)
|
|
503
564
|
compiled: CompiledProject = state.compiled
|
|
504
565
|
out: list[EnvironmentInfo] = []
|
|
505
566
|
promoted_ats = await state.store.environment_promoted_at()
|
|
@@ -555,10 +616,16 @@ async def rollback_environment_endpoint(name: FromPath[str], state: State, data:
|
|
|
555
616
|
|
|
556
617
|
|
|
557
618
|
@get("/runs")
|
|
558
|
-
async def get_runs(state: State) -> list[RunInfo]:
|
|
619
|
+
async def get_runs(state: State, limit: FromQuery[int | None] = None) -> list[RunInfo]:
|
|
559
620
|
runs: list[RunInfo] = []
|
|
560
621
|
for run in await state.store.list_runs():
|
|
561
622
|
partition = [str(run["partition_start"]), str(run["partition_end"])] if run["partition_start"] else None
|
|
623
|
+
started, finished = run.get("started_at"), run.get("finished_at")
|
|
624
|
+
duration = None
|
|
625
|
+
if started and finished: # ledger timestamps are naive ISO; the span is what we want
|
|
626
|
+
from datetime import datetime
|
|
627
|
+
|
|
628
|
+
duration = (datetime.fromisoformat(finished) - datetime.fromisoformat(started)).total_seconds()
|
|
562
629
|
runs.append(
|
|
563
630
|
RunInfo(
|
|
564
631
|
id=run["id"],
|
|
@@ -571,9 +638,11 @@ async def get_runs(state: State) -> list[RunInfo]:
|
|
|
571
638
|
partition=partition,
|
|
572
639
|
restate=run["restate"],
|
|
573
640
|
idempotency_key=run["idempotency_key"],
|
|
641
|
+
environment=run.get("environment"),
|
|
642
|
+
duration=duration,
|
|
574
643
|
)
|
|
575
644
|
)
|
|
576
|
-
return runs
|
|
645
|
+
return runs[:limit] if limit else runs # list_runs is newest-first
|
|
577
646
|
|
|
578
647
|
|
|
579
648
|
@get("/runs/{run_id:int}")
|
|
@@ -581,10 +650,12 @@ async def get_run(run_id: FromPath[int], state: State) -> RunDetail:
|
|
|
581
650
|
run = await state.store.get_run(run_id)
|
|
582
651
|
if run is None:
|
|
583
652
|
raise NotFoundException(detail=f"unknown run: {run_id}")
|
|
584
|
-
# lifecycle events are keyed by run id (worker) and idempotency key (enqueue)
|
|
585
|
-
events
|
|
653
|
+
# lifecycle events are keyed by run id (worker) and idempotency key (enqueue); the
|
|
654
|
+
# per-model events are keyed by payload.run (their entity is the model name)
|
|
655
|
+
events = await state.store.events_for_entity(str(run_id)) + await state.store.events_for_run(run_id)
|
|
586
656
|
if run["idempotency_key"]:
|
|
587
|
-
events
|
|
657
|
+
events += await state.store.events_for_entity(run["idempotency_key"])
|
|
658
|
+
events = sorted({event["seq"]: event for event in events}.values(), key=lambda event: event["seq"])
|
|
588
659
|
partition = [str(run["partition_start"]), str(run["partition_end"])] if run["partition_start"] else None
|
|
589
660
|
return RunDetail(
|
|
590
661
|
id=run["id"],
|
|
@@ -603,6 +674,7 @@ async def get_run(run_id: FromPath[int], state: State) -> RunDetail:
|
|
|
603
674
|
|
|
604
675
|
@post("/runs", opt={"scope": "write"})
|
|
605
676
|
async def create_run(data: CreateRun, state: State) -> CreateRunResult:
|
|
677
|
+
await reload_if_stale(state)
|
|
606
678
|
compiled: CompiledProject = state.compiled
|
|
607
679
|
env = data.environment or state.environment
|
|
608
680
|
try:
|
|
@@ -632,6 +704,7 @@ async def create_run(data: CreateRun, state: State) -> CreateRunResult:
|
|
|
632
704
|
enqueued = await state.store.enqueue_run(key, models, partition, 0, restate=data.restate)
|
|
633
705
|
if enqueued:
|
|
634
706
|
await state.store.append_event("run.enqueued", entity=key, payload={"models": models})
|
|
707
|
+
state.drain_wanted.set() # wake the scheduler now, don't wait out the tick interval
|
|
635
708
|
return CreateRunResult(enqueued=1 if enqueued else 0, models=models)
|
|
636
709
|
|
|
637
710
|
|
|
@@ -648,6 +721,7 @@ async def cancel_run(run_id: FromPath[int], state: State) -> dict:
|
|
|
648
721
|
|
|
649
722
|
@post("/apply", opt={"scope": "write"})
|
|
650
723
|
async def post_apply(data: ApplyRequest, state: State) -> ApplyResponse:
|
|
724
|
+
await reload_if_stale(state)
|
|
651
725
|
compiled: CompiledProject = state.compiled
|
|
652
726
|
env = data.environment or state.environment
|
|
653
727
|
try:
|
|
@@ -713,12 +787,28 @@ async def post_apply(data: ApplyRequest, state: State) -> ApplyResponse:
|
|
|
713
787
|
for name, c in result.rows.items()
|
|
714
788
|
},
|
|
715
789
|
timings={name: round(seconds, 3) for name, seconds in result.timings.items()},
|
|
790
|
+
gated=result.gated,
|
|
791
|
+
checks=[
|
|
792
|
+
CheckOutcomeInfo(
|
|
793
|
+
model=outcome.model,
|
|
794
|
+
name=outcome.name,
|
|
795
|
+
check_type=outcome.type,
|
|
796
|
+
severity=outcome.severity,
|
|
797
|
+
status=outcome.status,
|
|
798
|
+
failures=outcome.failures,
|
|
799
|
+
message=outcome.message,
|
|
800
|
+
)
|
|
801
|
+
for outcome in result.checks
|
|
802
|
+
],
|
|
716
803
|
)
|
|
717
804
|
|
|
718
805
|
|
|
719
806
|
@get("/checks")
|
|
720
|
-
async def get_checks(
|
|
721
|
-
|
|
807
|
+
async def get_checks(
|
|
808
|
+
state: State, model: FromQuery[str | None] = None, limit: FromQuery[int | None] = None
|
|
809
|
+
) -> list[CheckResultInfo]:
|
|
810
|
+
rows = await state.store.list_check_results(model)
|
|
811
|
+
return [CheckResultInfo(**row) for row in (rows[:limit] if limit else rows)]
|
|
722
812
|
|
|
723
813
|
|
|
724
814
|
async def _enqueue_stream_consumers(state: State, stream: StreamDef) -> None:
|
|
@@ -734,6 +824,7 @@ async def _enqueue_stream_consumers(state: State, stream: StreamDef) -> None:
|
|
|
734
824
|
key = f"stream:{stream.name}:{watermark}"
|
|
735
825
|
if await state.store.enqueue_run(key, sorted(consumers), None, 0):
|
|
736
826
|
await state.store.append_event("run.enqueued", entity=key, payload={"models": sorted(consumers)})
|
|
827
|
+
state.drain_wanted.set() # wake the scheduler now, don't wait out the tick interval
|
|
737
828
|
|
|
738
829
|
|
|
739
830
|
def _stream_or_404(state: State, name: str) -> StreamDef:
|
|
@@ -938,6 +1029,7 @@ async def get_schedules(state: State) -> list[ScheduleInfo]:
|
|
|
938
1029
|
|
|
939
1030
|
from cronsim import CronSim
|
|
940
1031
|
|
|
1032
|
+
await reload_if_stale(state)
|
|
941
1033
|
compiled: CompiledProject = state.compiled
|
|
942
1034
|
infos: list[ScheduleInfo] = []
|
|
943
1035
|
for name in sorted(compiled.models):
|
|
@@ -1009,12 +1101,13 @@ async def _described_columns(state: State, name: str, promoted: dict[str, str],
|
|
|
1009
1101
|
|
|
1010
1102
|
|
|
1011
1103
|
@get("/lineage")
|
|
1012
|
-
async def get_lineage(state: State) -> LineageResponse:
|
|
1104
|
+
async def get_lineage(state: State, environment: FromQuery[str | None] = None) -> LineageResponse:
|
|
1013
1105
|
"""The whole graph in one payload: nodes (with warehouse-described column
|
|
1014
1106
|
types), table edges, column lineage, and stream sources — the UI renders
|
|
1015
|
-
and traces without a request per node."""
|
|
1107
|
+
and traces without a request per node. ``?environment=`` inspects a sandbox."""
|
|
1108
|
+
await reload_if_stale(state)
|
|
1016
1109
|
compiled: CompiledProject = state.compiled
|
|
1017
|
-
promoted = await state.store.get_environment(state.environment)
|
|
1110
|
+
promoted = await state.store.get_environment(environment or state.environment)
|
|
1018
1111
|
snapshots = await state.store.get_snapshots(promoted.items())
|
|
1019
1112
|
|
|
1020
1113
|
order = compiled.graph.topological_sort()
|
|
@@ -1089,6 +1182,7 @@ async def post_checks_run(state: State, data: RunChecksRequest | None = None) ->
|
|
|
1089
1182
|
from interlace.checks.runner import run_checks
|
|
1090
1183
|
|
|
1091
1184
|
request = data or RunChecksRequest()
|
|
1185
|
+
await reload_if_stale(state)
|
|
1092
1186
|
compiled: CompiledProject = state.compiled
|
|
1093
1187
|
env = request.environment or state.environment
|
|
1094
1188
|
promoted = await state.store.get_environment(env)
|
|
@@ -1319,11 +1413,17 @@ def create_app(
|
|
|
1319
1413
|
app.state.engines = engines
|
|
1320
1414
|
app.state.environment = environment
|
|
1321
1415
|
app.state.root = project.root
|
|
1416
|
+
app.state.model_paths = project.config.model_paths # for the on-demand recompile staleness probe
|
|
1417
|
+
app.state.reload_lock = asyncio.Lock() # serialise recompiles; recompile at most once per change
|
|
1418
|
+
app.state.source_mtime = _source_mtime(project.root, project.config.model_paths)
|
|
1322
1419
|
app.state.parallelism = project.config.parallelism
|
|
1323
1420
|
app.state.engine_configs = project.config.engine_configs()
|
|
1324
1421
|
app.state.default_engine = project.config.default_engine
|
|
1325
1422
|
app.state.describe_cache = {} # (model, fingerprint) -> {column: type}, filled by /lineage
|
|
1326
1423
|
app.state.apply_lock = asyncio.Lock() # serialise applies against the single warehouse connection
|
|
1424
|
+
# Wake the scheduler's drain the instant a run is enqueued, instead of waiting
|
|
1425
|
+
# out the tick interval — an enqueue from the UI/API/stream picks up promptly.
|
|
1426
|
+
app.state.drain_wanted = asyncio.Event()
|
|
1327
1427
|
app.state.streams = streams
|
|
1328
1428
|
app.state.stream_log = stream_log
|
|
1329
1429
|
app.state.flush_targets = flush_targets # streams + their quarantine shadows
|
|
@@ -1411,11 +1511,15 @@ def create_app(
|
|
|
1411
1511
|
app.state.flush_wanted.set() # catch up anything durable but unflushed at last shutdown
|
|
1412
1512
|
|
|
1413
1513
|
async def scheduler_loop() -> None:
|
|
1414
|
-
trigger_engine = TriggerEngine(build_triggers(compiled), store)
|
|
1415
1514
|
next_trim = asyncio.get_running_loop().time() # first tick trims; then every 6h
|
|
1416
1515
|
while True:
|
|
1516
|
+
# cleared before the pass so an enqueue landing mid-drain re-arms the
|
|
1517
|
+
# event and earns another prompt pass rather than being lost
|
|
1518
|
+
app.state.drain_wanted.clear()
|
|
1417
1519
|
try:
|
|
1418
|
-
await
|
|
1520
|
+
await reload_if_stale(app.state) # scheduled/queued builds see on-disk edits too
|
|
1521
|
+
compiled = app.state.compiled # live: reload may have swapped it
|
|
1522
|
+
await TriggerEngine(build_triggers(compiled), store).tick(datetime.now())
|
|
1419
1523
|
if asyncio.get_running_loop().time() >= next_trim:
|
|
1420
1524
|
# event_log / check_results / terminal queue rows grow with
|
|
1421
1525
|
# every apply and flush; nothing else reclaims them
|
|
@@ -1435,7 +1539,9 @@ def create_app(
|
|
|
1435
1539
|
await sweep_streams(streams.values(), stream_log, engine) # apply retention
|
|
1436
1540
|
except Exception:
|
|
1437
1541
|
logger.exception("scheduler tick failed; retrying next interval")
|
|
1438
|
-
|
|
1542
|
+
# wake early the instant a run is enqueued; otherwise tick on the interval
|
|
1543
|
+
with contextlib.suppress(asyncio.TimeoutError):
|
|
1544
|
+
await asyncio.wait_for(app.state.drain_wanted.wait(), timeout=scheduler_interval)
|
|
1439
1545
|
|
|
1440
1546
|
async def shutdown_watch() -> None:
|
|
1441
1547
|
"""End every open SSE stream the moment uvicorn begins shutting down.
|
|
@@ -74,6 +74,7 @@ input, textarea, select { font: inherit; color: inherit; }
|
|
|
74
74
|
}
|
|
75
75
|
.brand { display: flex; align-items: center; gap: 8px; }
|
|
76
76
|
.wordmark { font-weight: 600; letter-spacing: 0.02em; }
|
|
77
|
+
.wordmark .wordmark-tld { color: var(--tx-faint); font-weight: 500; } /* muted ".sh" */
|
|
77
78
|
.grow { flex: 1; }
|
|
78
79
|
|
|
79
80
|
.env-chip {
|
|
@@ -181,6 +182,26 @@ table.t { width: 100%; border-collapse: collapse; font-size: 12px; }
|
|
|
181
182
|
.t .dim { color: var(--tx-dim); }
|
|
182
183
|
.t .faint { color: var(--tx-faint); }
|
|
183
184
|
|
|
185
|
+
/* compact variant (run detail): the CLI's small font + rule-under-header, but no
|
|
186
|
+
per-row separators — the rows read as one tight block, not a striped grid */
|
|
187
|
+
.t.compact { font-size: 11.5px; }
|
|
188
|
+
.t.compact th { padding: 3px 9px; }
|
|
189
|
+
.t.compact td { padding: 2.5px 9px; border-bottom: 0; }
|
|
190
|
+
.t.compact .tl-mark { display: inline-block; width: 13px; text-align: center; }
|
|
191
|
+
|
|
192
|
+
/* run detail: a dim header line (env · duration · models · window · attempt) above
|
|
193
|
+
the build table, all riding the expand-row's inset background. .dim/.faint are
|
|
194
|
+
otherwise scoped to .t, so re-declare them for the header spans living outside it */
|
|
195
|
+
.run-detail { display: flex; flex-direction: column; gap: 8px; }
|
|
196
|
+
.run-meta { font-size: 11.5px; }
|
|
197
|
+
.run-detail .dim { color: var(--tx-dim); }
|
|
198
|
+
.run-detail .faint { color: var(--tx-faint); }
|
|
199
|
+
|
|
200
|
+
/* models cell: "13 models · a, b, c…" — count stays, names shrink to an ellipsis */
|
|
201
|
+
.run-models { display: inline-flex; align-items: baseline; max-width: 460px; vertical-align: top; }
|
|
202
|
+
.rm-count { flex: 0 0 auto; white-space: nowrap; }
|
|
203
|
+
.rm-names { flex: 0 1 auto; min-width: 0; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; color: var(--tx-dim); }
|
|
204
|
+
|
|
184
205
|
.empty { padding: 26px; text-align: center; color: var(--tx-dim); }
|
|
185
206
|
.empty .hint { font-size: 11px; color: var(--tx-faint); margin-top: 4px; }
|
|
186
207
|
|
|
@@ -324,6 +345,24 @@ pre.sql { background: var(--ink-0); border: 1px solid var(--line-soft); border-r
|
|
|
324
345
|
.feed-row .ty { color: var(--tx-dim); width: 140px; flex-shrink: 0; }
|
|
325
346
|
.feed-row .en { color: var(--tx); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
|
326
347
|
|
|
348
|
+
/* ---- activity: build episodes (apply / run) + model timelines ---------------- */
|
|
349
|
+
.episode { border-bottom: 1px solid var(--line-soft); }
|
|
350
|
+
.episode:last-child { border-bottom: 0; }
|
|
351
|
+
.episode-head { display: flex; align-items: center; gap: 8px; width: 100%; text-align: left; padding: 5px 12px; font-size: 11.5px; }
|
|
352
|
+
.episode-head:hover { background: var(--ink-2); }
|
|
353
|
+
.episode-head .spread { flex: 1; }
|
|
354
|
+
.ep-kind { color: var(--tx); font-weight: 500; }
|
|
355
|
+
.ep-target { color: var(--violet); }
|
|
356
|
+
.ep-meta { color: var(--tx-dim); font-size: 11px; }
|
|
357
|
+
.episode-head .ts { color: var(--tx-faint); }
|
|
358
|
+
.timeline { display: flex; flex-direction: column; gap: 2px; padding: 4px 12px 8px 30px; }
|
|
359
|
+
.tl-step { display: flex; align-items: baseline; gap: 8px; font-size: 11.5px; }
|
|
360
|
+
.tl-mark { width: 13px; text-align: center; flex-shrink: 0; }
|
|
361
|
+
.tl-mark.running::before { content: "◌"; color: var(--amber); display: inline-block; animation: spin 0.9s linear infinite; }
|
|
362
|
+
.tl-name { flex: 1; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
|
363
|
+
.tl-dur { color: var(--tx-faint); font-size: 11px; }
|
|
364
|
+
.tl-empty { padding: 8px 12px; color: var(--tx-faint); font-size: 11.5px; }
|
|
365
|
+
|
|
327
366
|
/* ---- query console --------------------------------------------------------- */
|
|
328
367
|
.q-layout { display: grid; grid-template-columns: 1fr 240px; gap: 12px; align-items: start; }
|
|
329
368
|
.q-layout .card { margin-top: 0; }
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64" width="64" height="64" fill="none" role="img" aria-label="Interlace">
|
|
2
|
+
<title>Interlace</title>
|
|
3
|
+
<!-- Two loops in plain weave (matches interlace.sh). Each loop is knocked out by the
|
|
4
|
+
other's own stroke at width 13 against the 6 drawn, so the clearance follows the
|
|
5
|
+
curve; the clip paths confine each cut to one crossing quadrant. -->
|
|
6
|
+
<style>
|
|
7
|
+
.a { stroke: #6D28D9; } .b { stroke: #8B5CF6; }
|
|
8
|
+
@media (prefers-color-scheme: dark) { .a { stroke: #8B5CF6; } .b { stroke: #C4B5FD; } }
|
|
9
|
+
</style>
|
|
10
|
+
<defs>
|
|
11
|
+
<mask id="dh" maskUnits="userSpaceOnUse" x="0" y="0" width="64" height="64">
|
|
12
|
+
<rect width="64" height="64" fill="#fff"/>
|
|
13
|
+
<g clip-path="url(#dtl)"><rect x="20" y="6" width="24" height="52" rx="12" fill="none" stroke="#000" stroke-width="13"/></g>
|
|
14
|
+
<g clip-path="url(#dbr)"><rect x="20" y="6" width="24" height="52" rx="12" fill="none" stroke="#000" stroke-width="13"/></g>
|
|
15
|
+
</mask>
|
|
16
|
+
<mask id="dv" maskUnits="userSpaceOnUse" x="0" y="0" width="64" height="64">
|
|
17
|
+
<rect width="64" height="64" fill="#fff"/>
|
|
18
|
+
<g clip-path="url(#dtr)"><rect x="6" y="20" width="52" height="24" rx="12" fill="none" stroke="#000" stroke-width="13"/></g>
|
|
19
|
+
<g clip-path="url(#dbl)"><rect x="6" y="20" width="52" height="24" rx="12" fill="none" stroke="#000" stroke-width="13"/></g>
|
|
20
|
+
</mask>
|
|
21
|
+
<clipPath id="dtl"><rect x="10" y="10" width="18" height="18"/></clipPath>
|
|
22
|
+
<clipPath id="dbr"><rect x="36" y="36" width="18" height="18"/></clipPath>
|
|
23
|
+
<clipPath id="dtr"><rect x="36" y="10" width="18" height="18"/></clipPath>
|
|
24
|
+
<clipPath id="dbl"><rect x="10" y="36" width="18" height="18"/></clipPath>
|
|
25
|
+
</defs>
|
|
26
|
+
<rect x="6" y="20" width="52" height="24" rx="12" fill="none" class="a" stroke="#8B5CF6" stroke-width="6" mask="url(#dh)"/>
|
|
27
|
+
<rect x="20" y="6" width="24" height="52" rx="12" fill="none" class="b" stroke="#C4B5FD" stroke-width="6" mask="url(#dv)"/>
|
|
28
|
+
</svg>
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
<head>
|
|
4
4
|
<meta charset="utf-8">
|
|
5
5
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
6
|
-
<title>interlace</title>
|
|
6
|
+
<title>interlace.sh</title>
|
|
7
7
|
<meta name="description" content="interlace control plane — a Python/SQL-first data platform: transformation, orchestration, and durable streaming in one daemon.">
|
|
8
8
|
|
|
9
9
|
<meta name="theme-color" content="#0B0E16">
|
|
@@ -14,8 +14,8 @@
|
|
|
14
14
|
<div class="shell">
|
|
15
15
|
<header class="topbar">
|
|
16
16
|
<a class="brand" href="#/overview" title="Overview">
|
|
17
|
-
<svg class="mark" width="
|
|
18
|
-
<span class="wordmark">interlace</span>
|
|
17
|
+
<svg class="mark" width="22" height="22" viewBox="0 0 64 64" fill="none" aria-hidden="true"><defs><mask id="ilh" maskUnits="userSpaceOnUse" x="0" y="0" width="64" height="64"><rect width="64" height="64" fill="#fff"/><g clip-path="url(#iltl)"><rect x="20" y="6" width="24" height="52" rx="12" fill="none" stroke="#000" stroke-width="13"/></g><g clip-path="url(#ilbr)"><rect x="20" y="6" width="24" height="52" rx="12" fill="none" stroke="#000" stroke-width="13"/></g></mask><mask id="ilv" maskUnits="userSpaceOnUse" x="0" y="0" width="64" height="64"><rect width="64" height="64" fill="#fff"/><g clip-path="url(#iltr)"><rect x="6" y="20" width="52" height="24" rx="12" fill="none" stroke="#000" stroke-width="13"/></g><g clip-path="url(#ilbl)"><rect x="6" y="20" width="52" height="24" rx="12" fill="none" stroke="#000" stroke-width="13"/></g></mask><clipPath id="iltl"><rect x="10" y="10" width="18" height="18"/></clipPath><clipPath id="ilbr"><rect x="36" y="36" width="18" height="18"/></clipPath><clipPath id="iltr"><rect x="36" y="10" width="18" height="18"/></clipPath><clipPath id="ilbl"><rect x="10" y="36" width="18" height="18"/></clipPath></defs><rect x="6" y="20" width="52" height="24" rx="12" fill="none" stroke="#8B5CF6" stroke-width="6" mask="url(#ilh)"/><rect x="20" y="6" width="24" height="52" rx="12" fill="none" stroke="#C4B5FD" stroke-width="6" mask="url(#ilv)"/></svg>
|
|
18
|
+
<span class="wordmark">interlace<span class="wordmark-tld">.sh</span></span>
|
|
19
19
|
</a>
|
|
20
20
|
<button class="env-chip" id="envChip" title="Target environment (prod = the unprefixed namespace)">
|
|
21
21
|
<span class="env-key">env</span><span class="env-val" id="envName">prod</span>
|