interlaced 2.8.4__tar.gz → 2.8.7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {interlaced-2.8.4/src/interlaced.egg-info → interlaced-2.8.7}/PKG-INFO +1 -1
- {interlaced-2.8.4 → interlaced-2.8.7}/pyproject.toml +1 -1
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/cli/operate.py +3 -2
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/config/config.py +8 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/dsl/decorators.py +10 -1
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/plan/differ.py +12 -1
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/plan/schedule.py +5 -1
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/project.py +8 -3
- interlaced-2.8.7/src/interlace/scheduler/__init__.py +39 -0
- interlaced-2.8.7/src/interlace/scheduler/completion.py +132 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/scheduler/engine.py +4 -29
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/service/app.py +4 -3
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/service/lifespan.py +53 -8
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/service/routes/catalog.py +1 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/service/routes/models.py +7 -1
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/service/types.py +1 -1
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/service/ui/js/views/plan.js +4 -1
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/service/ui/js/views/system.js +1 -1
- interlaced-2.8.7/src/interlace/state/advisory.py +101 -0
- interlaced-2.8.7/src/interlace/state/conn.py +83 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/state/events.py +2 -2
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/state/locks.py +4 -4
- interlaced-2.8.7/src/interlace/state/pg.py +191 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/state/queue.py +8 -4
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/state/snapshots.py +3 -1
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/state/store.py +16 -9
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/streaming/log.py +33 -11
- {interlaced-2.8.4 → interlaced-2.8.7/src/interlaced.egg-info}/PKG-INFO +1 -1
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlaced.egg-info/SOURCES.txt +2 -0
- interlaced-2.8.4/src/interlace/scheduler/__init__.py +0 -17
- interlaced-2.8.4/src/interlace/state/advisory.py +0 -87
- interlaced-2.8.4/src/interlace/state/conn.py +0 -40
- {interlaced-2.8.4 → interlaced-2.8.7}/LICENSE +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/MANIFEST.in +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/README.md +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/setup.cfg +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/__init__.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/cdc/__init__.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/cdc/decode.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/cdc/publish.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/cdc/slot.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/checks/__init__.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/checks/builtin.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/checks/runner.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/checks/spec.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/cli/__init__.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/cli/build.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/cli/inspect.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/cli/main.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/cli/support.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/config/__init__.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/connections.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/contracts.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/dsl/__init__.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/dsl/discovery.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/dsl/dynamic.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/dsl/sql_config.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/engines/__init__.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/engines/adbc.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/engines/base.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/engines/bigquery.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/engines/duckdb.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/engines/postgres.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/engines/quack.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/engines/redshift.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/engines/registry.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/engines/snowflake.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/engines/spark.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/exceptions.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/graph/__init__.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/graph/column_lineage.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/graph/dag.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/graph/project.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/graph/selectors.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/inputs.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/inspect.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/ir/__init__.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/ir/canonicalize.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/ir/dialect.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/ir/fingerprint.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/ir/layout.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/ir/macros.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/ir/relation.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/ir/vars.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/mcp_server.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/physical/__init__.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/physical/changes.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/physical/ddl.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/physical/drift.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/physical/reconcile.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/physical/spec.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/plan/__init__.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/plan/annotate.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/plan/apply.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/plan/backfill.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/plan/comment.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/plan/delivery.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/plan/files.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/plan/fit.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/plan/orchestrate.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/plan/payload.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/plan/plan.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/plan/resolve.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/plan/result.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/plan/run.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/plan/table_diff.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/plan/transfer.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/py.typed +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/query.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/runtime/__init__.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/runtime/handles.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/runtime/python_model.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/scaffold.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/scheduler/daemon.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/scheduler/triggers.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/scheduler/worker.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/service/__init__.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/service/auth.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/service/present.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/service/routes/__init__.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/service/routes/admin.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/service/routes/environments.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/service/routes/plan.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/service/routes/runs.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/service/routes/streams.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/service/ui/app.css +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/service/ui/favicon.svg +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/service/ui/index.html +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/service/ui/js/api.js +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/service/ui/js/app.js +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/service/ui/js/dag.js +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/service/ui/js/timeline.js +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/service/ui/js/ui.js +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/service/ui/js/views/checks.js +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/service/ui/js/views/environments.js +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/service/ui/js/views/lineage.js +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/service/ui/js/views/models.js +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/service/ui/js/views/overview.js +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/service/ui/js/views/query.js +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/service/ui/js/views/runs.js +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/service/ui/js/views/streams.js +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/sinks.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/sources/__init__.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/sources/auth.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/sources/rest.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/state/__init__.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/state/beat.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/state/cdc.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/state/checks.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/state/codec.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/state/interval.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/state/janitor.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/state/keys.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/state/schema.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/state/snapshot.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/state/triggers.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/strategies/__init__.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/strategies/append.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/strategies/base.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/strategies/full_merge.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/strategies/hash_merge.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/strategies/incremental.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/strategies/merge.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/strategies/replace.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/strategies/replace_in_place.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/strategies/scd.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/strategies/view.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/streaming/__init__.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/streaming/materializer.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/streaming/schema.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/templates/cdc/README.md +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/templates/cdc/docker-compose.yml +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/templates/cdc/init/seed.sql +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/templates/cdc/interlace.yaml +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/templates/cdc/models/orders.sql +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/templates/cdc/models/orders_by_status.sql +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/templates/cdc/models/orders_stream.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/templates/cdc/push.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/templates/cdc/template.yaml +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/templates/events/README.md +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/templates/events/generate.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/templates/events/interlace.yaml +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/templates/events/models/events.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/templates/events/models/events_by_minute.sql +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/templates/events/models/events_by_type.sql +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/templates/events/models/top_users.sql +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/templates/events/models/user_spend.sql +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/templates/events/template.yaml +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/templates/github/README.md +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/templates/github/interlace.yaml +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/templates/github/models/github_issues.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/templates/github/models/issues_by_state.sql +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/templates/github/template.yaml +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/templates/postgres/README.md +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/templates/postgres/docker-compose.yml +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/templates/postgres/init/seed.sql +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/templates/postgres/interlace.yaml +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/templates/postgres/models/orders.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/templates/postgres/models/orders_by_status.sql +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/templates/postgres/template.yaml +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/templates/quickstart/README.md +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/templates/quickstart/interlace.yaml +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/templates/quickstart/models/enriched_events.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/templates/quickstart/models/event_summary.sql +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/templates/quickstart/models/raw_events.sql +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/templates/quickstart/template.yaml +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/testing/__init__.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlace/testing/golden.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlaced.egg-info/dependency_links.txt +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlaced.egg-info/entry_points.txt +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlaced.egg-info/requires.txt +0 -0
- {interlaced-2.8.4 → interlaced-2.8.7}/src/interlaced.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: interlaced
|
|
3
|
-
Version: 2.8.
|
|
3
|
+
Version: 2.8.7
|
|
4
4
|
Summary: Python and SQL models in one DAG. Transformation, orchestration and durable streaming in one process, on DuckDB and Postgres.
|
|
5
5
|
Author-email: Mark <mark@interlace.sh>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "interlaced"
|
|
7
|
-
version = "2.8.
|
|
7
|
+
version = "2.8.7"
|
|
8
8
|
description = "Python and SQL models in one DAG. Transformation, orchestration and durable streaming in one process, on DuckDB and Postgres."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.12"
|
|
@@ -288,7 +288,7 @@ def serve(
|
|
|
288
288
|
apply_on_start: bool = typer.Option(
|
|
289
289
|
True,
|
|
290
290
|
"--apply/--no-apply",
|
|
291
|
-
help="Apply the
|
|
291
|
+
help="Apply once at startup, while the API is already listening. --no-apply serves the warehouse as it is.",
|
|
292
292
|
),
|
|
293
293
|
allow_open: bool = typer.Option(
|
|
294
294
|
False,
|
|
@@ -299,7 +299,8 @@ def serve(
|
|
|
299
299
|
"""Run the interlace daemon: HTTP API + scheduler in one process (requires the `service` extra).
|
|
300
300
|
|
|
301
301
|
Use --no-scheduler for an API-only process (run `interlace scheduler` separately).
|
|
302
|
-
The project is applied once before the
|
|
302
|
+
The API listens immediately. The project is applied once at startup, before the
|
|
303
|
+
scheduler loop; pass --no-apply to skip that. Writes return 503 until the apply finishes.
|
|
303
304
|
A `cdc:` block in the project config is read here: each Postgres slot appends into its stream.
|
|
304
305
|
"""
|
|
305
306
|
try:
|
|
@@ -250,6 +250,9 @@ class ProjectConfig(BaseModel):
|
|
|
250
250
|
default_engine: str = "default"
|
|
251
251
|
engines: dict[str, EngineConfig] = Field(default_factory=dict)
|
|
252
252
|
state_path: str = ".interlace/state.db" # SQLite control-plane database
|
|
253
|
+
# When set, snapshots, the queue, and the event log live in this Postgres
|
|
254
|
+
# database (schema ``interlace``) instead of ``state_path``. Not the warehouse.
|
|
255
|
+
state_url: str | None = None
|
|
253
256
|
# Optional NDJSON mirror of the operator event log. Unset writes nowhere.
|
|
254
257
|
# One JSON line is appended after each SQLite commit (CLI and daemon share it).
|
|
255
258
|
event_log_path: str | None = None
|
|
@@ -275,6 +278,11 @@ class ProjectConfig(BaseModel):
|
|
|
275
278
|
secrets: dict[str, SecretConfig] = Field(default_factory=dict)
|
|
276
279
|
quack_token: str | None = None # token for quack: databases (or INTERLACE_QUACK_TOKEN)
|
|
277
280
|
stream_path: str = ".interlace/streams.db" # durable stream log (SQLite WAL)
|
|
281
|
+
# When set, the stream log lives in this Postgres database (schema
|
|
282
|
+
# ``interlace_streams``) instead of ``stream_path``. A commit is durable
|
|
283
|
+
# under Postgres ``synchronous_commit`` (the default), the same promise as
|
|
284
|
+
# the SQLite WAL fsync.
|
|
285
|
+
stream_url: str | None = None
|
|
278
286
|
# Databases to ATTACH to the warehouse engine at open: alias -> DuckDB attach
|
|
279
287
|
# URI/path (a .duckdb file, "postgres:...", "sqlite:...", ...). Models can read
|
|
280
288
|
# them and table exports can write to them as <alias>.<schema>.<table>.
|
|
@@ -32,7 +32,8 @@ ModelFn = Callable[..., Any]
|
|
|
32
32
|
|
|
33
33
|
|
|
34
34
|
def normalize_schedule(schedule: Mapping[str, Any] | None, *, model: str) -> dict[str, str] | None:
|
|
35
|
-
"""Schedules are string values. ``fresh: {column, within}`` becomes ``"column within"``.
|
|
35
|
+
"""Schedules are string values. ``fresh: {column, within}`` becomes ``"column within"``.
|
|
36
|
+
``after: [a, b]`` becomes ``"a,b"``."""
|
|
36
37
|
if not schedule:
|
|
37
38
|
return None
|
|
38
39
|
normalised: dict[str, str] = {}
|
|
@@ -46,6 +47,14 @@ def normalize_schedule(schedule: Mapping[str, Any] | None, *, model: str) -> dic
|
|
|
46
47
|
raise DefinitionError(f"model {model!r}: fresh.within must be a window like '2h'")
|
|
47
48
|
normalised[key] = f"{column.strip()} {window.strip()}"
|
|
48
49
|
continue
|
|
50
|
+
if key == "after" and isinstance(value, (list, tuple)):
|
|
51
|
+
names = [str(item).strip() for item in value]
|
|
52
|
+
if not names or any(not name for name in names):
|
|
53
|
+
raise DefinitionError(f"model {model!r}: after must name a model, or a list of models")
|
|
54
|
+
normalised[key] = ",".join(names)
|
|
55
|
+
continue
|
|
56
|
+
if key == "after" and not isinstance(value, str):
|
|
57
|
+
raise DefinitionError(f"model {model!r}: after must name a model, or a list of models")
|
|
49
58
|
if not isinstance(value, str):
|
|
50
59
|
raise DefinitionError(f"model {model!r}: schedule {key!r} must be a string")
|
|
51
60
|
normalised[key] = value
|
|
@@ -338,6 +338,8 @@ async def diff( # noqa: C901
|
|
|
338
338
|
plan = Plan(environment=environment)
|
|
339
339
|
impact: dict[str, str] = {} # changed models only: "semantic" | "additive" | "clean"
|
|
340
340
|
touched: dict[str, frozenset[str] | None] = {} # semantic models: provably-changed columns (None = all)
|
|
341
|
+
# downstream models: the upstream columns that forced the rebuild
|
|
342
|
+
indirect: dict[str, frozenset[str]] = {}
|
|
341
343
|
|
|
342
344
|
previous_snapshots = await state.get_snapshots(
|
|
343
345
|
(name, fingerprint)
|
|
@@ -410,6 +412,8 @@ async def diff( # noqa: C901
|
|
|
410
412
|
consumed = _consumed_columns(model.ast, dep)
|
|
411
413
|
if dep_touched is not None and consumed is not None and not (consumed & dep_touched):
|
|
412
414
|
continue # column-pruned: reads only provably-unchanged columns
|
|
415
|
+
if dep_touched is not None and consumed is not None:
|
|
416
|
+
indirect[model.name] = consumed & dep_touched
|
|
413
417
|
verdict = "semantic"
|
|
414
418
|
break
|
|
415
419
|
if dep_impact == "additive" and _selects_star(model.ast):
|
|
@@ -436,8 +440,15 @@ async def diff( # noqa: C901
|
|
|
436
440
|
)
|
|
437
441
|
if inherit:
|
|
438
442
|
category = ChangeCategory.FORWARD_ONLY
|
|
443
|
+
shown = added
|
|
444
|
+
if impact.get(model.name) == "semantic":
|
|
445
|
+
own = touched.get(model.name)
|
|
446
|
+
if own:
|
|
447
|
+
shown = tuple(sorted(own))
|
|
448
|
+
elif model.name in indirect:
|
|
449
|
+
shown = tuple(sorted(indirect[model.name]))
|
|
439
450
|
plan.changes.append(
|
|
440
|
-
ModelChange(model.name, ChangeType.MODIFIED, category, previous_fingerprint, model.fingerprint,
|
|
451
|
+
ModelChange(model.name, ChangeType.MODIFIED, category, previous_fingerprint, model.fingerprint, shown)
|
|
441
452
|
)
|
|
442
453
|
if inherit: # copy-on-write: history seeds the NEW table; checks gate before views move
|
|
443
454
|
snapshot = replace(
|
|
@@ -25,7 +25,8 @@ from interlace.ir.relation import TableRef
|
|
|
25
25
|
from interlace.plan.backfill import run_backfill
|
|
26
26
|
from interlace.plan.plan import BackfillTask, Plan
|
|
27
27
|
from interlace.plan.result import ApplyResult, ProgressCallback, build_detail, failure_detail
|
|
28
|
-
from interlace.
|
|
28
|
+
from interlace.scheduler.completion import after_waiters, wake_after
|
|
29
|
+
from interlace.state.store import SqliteStateStore, StateStore
|
|
29
30
|
|
|
30
31
|
logger = logging.getLogger("interlace.apply")
|
|
31
32
|
|
|
@@ -123,6 +124,7 @@ async def schedule_builds( # noqa: C901
|
|
|
123
124
|
deps.add(ref)
|
|
124
125
|
blocking[name] = deps
|
|
125
126
|
|
|
127
|
+
waiting = after_waiters(compiled)
|
|
126
128
|
finished = {name: asyncio.Event() for name in per_model}
|
|
127
129
|
build_slots = asyncio.Semaphore(max(1, parallelism))
|
|
128
130
|
|
|
@@ -163,6 +165,8 @@ async def schedule_builds( # noqa: C901
|
|
|
163
165
|
finished[name].set()
|
|
164
166
|
if on_progress is not None:
|
|
165
167
|
on_progress(name, "done", build_detail(result, name))
|
|
168
|
+
if isinstance(state, SqliteStateStore):
|
|
169
|
+
await wake_after(state, compiled, name, building=set(per_model), waiters=waiting)
|
|
166
170
|
|
|
167
171
|
# Copied into each build task at creation, so a Python model that registers
|
|
168
172
|
# models (including from a worker thread) appends to this same list.
|
|
@@ -375,15 +375,20 @@ class Project:
|
|
|
375
375
|
)
|
|
376
376
|
|
|
377
377
|
async def open_state(self) -> SqliteStateStore:
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
378
|
+
if self.config.state_url:
|
|
379
|
+
store = await SqliteStateStore.open_postgres(self.config.state_url)
|
|
380
|
+
else:
|
|
381
|
+
path = self.root / self.config.state_path
|
|
382
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
383
|
+
store = await SqliteStateStore.open(path)
|
|
381
384
|
if self.config.event_log_path:
|
|
382
385
|
target = Path(self.config.event_log_path)
|
|
383
386
|
store.event_log_path = str(target if target.is_absolute() else self.root / target)
|
|
384
387
|
return store
|
|
385
388
|
|
|
386
389
|
async def open_stream_log(self) -> SqliteStreamLog:
|
|
390
|
+
if self.config.stream_url:
|
|
391
|
+
return await SqliteStreamLog.open_postgres(self.config.stream_url)
|
|
387
392
|
path = self.root / self.config.stream_path
|
|
388
393
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
389
394
|
return await SqliteStreamLog.open(path)
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"""Orchestration: triggers, the trigger engine, and the worker (queue = state store).
|
|
2
|
+
|
|
3
|
+
Submodules import each other. This package does not load them up front: importing
|
|
4
|
+
``interlace.scheduler.completion`` must not pull in the worker, which imports the
|
|
5
|
+
apply path that imports completion.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import importlib
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
__all__ = [
|
|
14
|
+
"CronTrigger",
|
|
15
|
+
"IntervalTrigger",
|
|
16
|
+
"RunRequest",
|
|
17
|
+
"Trigger",
|
|
18
|
+
"TriggerEngine",
|
|
19
|
+
"build_triggers",
|
|
20
|
+
"drain",
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
_EXPORTS: dict[str, tuple[str, str]] = {
|
|
24
|
+
"CronTrigger": ("interlace.scheduler.triggers", "CronTrigger"),
|
|
25
|
+
"IntervalTrigger": ("interlace.scheduler.triggers", "IntervalTrigger"),
|
|
26
|
+
"RunRequest": ("interlace.scheduler.triggers", "RunRequest"),
|
|
27
|
+
"Trigger": ("interlace.scheduler.triggers", "Trigger"),
|
|
28
|
+
"TriggerEngine": ("interlace.scheduler.engine", "TriggerEngine"),
|
|
29
|
+
"build_triggers": ("interlace.scheduler.engine", "build_triggers"),
|
|
30
|
+
"drain": ("interlace.scheduler.worker", "drain"),
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def __getattr__(name: str) -> Any:
|
|
35
|
+
found = _EXPORTS.get(name)
|
|
36
|
+
if found is None:
|
|
37
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
38
|
+
module_name, attr = found
|
|
39
|
+
return getattr(importlib.import_module(module_name), attr)
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
"""Run a model when another model finishes.
|
|
2
|
+
|
|
3
|
+
``schedule: {after: …}`` is not a clock tick. Apply calls :func:`wake_after` once
|
|
4
|
+
a build reaches ``model.done``. The trigger engine only validates the graph.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import time
|
|
10
|
+
|
|
11
|
+
from interlace.exceptions import DefinitionError
|
|
12
|
+
from interlace.graph.project import CompiledProject
|
|
13
|
+
from interlace.state.store import SqliteStateStore
|
|
14
|
+
|
|
15
|
+
_SCHEDULE_KINDS = ("cron", "every", "watch", "on_change", "fresh", "after", "webhook")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def schedule_kind(model: str, schedule: dict[str, str]) -> tuple[str, str]:
|
|
19
|
+
"""The one schedule key and its value.
|
|
20
|
+
|
|
21
|
+
Cron, interval, file watch, table change, freshness, upstream completion, or webhook.
|
|
22
|
+
"""
|
|
23
|
+
present = [key for key in _SCHEDULE_KINDS if key in schedule]
|
|
24
|
+
if len(present) != 1:
|
|
25
|
+
raise DefinitionError(
|
|
26
|
+
f"model {model!r}: schedule needs exactly one of {', '.join(_SCHEDULE_KINDS)}",
|
|
27
|
+
details={"schedule": schedule},
|
|
28
|
+
)
|
|
29
|
+
return present[0], schedule[present[0]]
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def scheduled_closure(project: CompiledProject, models: list[str]) -> list[str]:
|
|
33
|
+
"""The triggered models plus every model downstream of them.
|
|
34
|
+
|
|
35
|
+
A cron, interval, file watch, table change, freshness, upstream completion, or webhook
|
|
36
|
+
means that model's inputs changed. Downstream snapshots would otherwise keep the
|
|
37
|
+
previous build. An explicit ``run --select`` is not expanded here: ``model``,
|
|
38
|
+
``model+``, and ``+model`` stay exactly what was asked for.
|
|
39
|
+
"""
|
|
40
|
+
chosen: set[str] = set()
|
|
41
|
+
for name in models:
|
|
42
|
+
chosen.add(name)
|
|
43
|
+
if name in project.models:
|
|
44
|
+
chosen |= project.graph.descendants(name)
|
|
45
|
+
return sorted(chosen)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def after_waiters(project: CompiledProject) -> dict[str, list[str]]:
|
|
49
|
+
"""Map an upstream model to the models that run when it finishes.
|
|
50
|
+
|
|
51
|
+
``schedule: {after: raw}`` or ``{after: [raw, staging]}`` (stored as
|
|
52
|
+
``"raw,staging"``). A model cannot name itself, and the ``after`` edges
|
|
53
|
+
cannot cycle — otherwise two models would enqueue each other forever.
|
|
54
|
+
"""
|
|
55
|
+
found: dict[str, list[str]] = {}
|
|
56
|
+
for model in project.models.values():
|
|
57
|
+
if not model.schedule or "after" not in model.schedule:
|
|
58
|
+
continue
|
|
59
|
+
kind, value = schedule_kind(model.name, model.schedule)
|
|
60
|
+
if kind != "after":
|
|
61
|
+
continue
|
|
62
|
+
names = list(dict.fromkeys(part.strip() for part in value.split(",") if part.strip()))
|
|
63
|
+
if not names:
|
|
64
|
+
raise DefinitionError(f"model {model.name!r}: after must name a model")
|
|
65
|
+
if model.name in names:
|
|
66
|
+
raise DefinitionError(f"model {model.name!r}: after cannot name itself")
|
|
67
|
+
missing = [name for name in names if name not in project.models]
|
|
68
|
+
if missing:
|
|
69
|
+
raise DefinitionError(
|
|
70
|
+
f"model {model.name!r}: after names unknown models: {', '.join(missing)}",
|
|
71
|
+
details={"missing": missing},
|
|
72
|
+
)
|
|
73
|
+
for name in names:
|
|
74
|
+
found.setdefault(name, []).append(model.name)
|
|
75
|
+
_reject_after_cycles(found)
|
|
76
|
+
return found
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _reject_after_cycles(waiters: dict[str, list[str]]) -> None:
|
|
80
|
+
visiting: set[str] = set()
|
|
81
|
+
visited: set[str] = set()
|
|
82
|
+
|
|
83
|
+
def walk(node: str) -> None:
|
|
84
|
+
if node in visited:
|
|
85
|
+
return
|
|
86
|
+
if node in visiting:
|
|
87
|
+
raise DefinitionError(f"schedule after has a cycle through {node!r}")
|
|
88
|
+
visiting.add(node)
|
|
89
|
+
for nxt in waiters.get(node, ()):
|
|
90
|
+
walk(nxt)
|
|
91
|
+
visiting.remove(node)
|
|
92
|
+
visited.add(node)
|
|
93
|
+
|
|
94
|
+
for node in list(waiters):
|
|
95
|
+
walk(node)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
async def wake_after(
|
|
99
|
+
store: SqliteStateStore,
|
|
100
|
+
project: CompiledProject,
|
|
101
|
+
upstream: str,
|
|
102
|
+
*,
|
|
103
|
+
building: set[str],
|
|
104
|
+
waiters: dict[str, list[str]] | None = None,
|
|
105
|
+
stamp: str | None = None,
|
|
106
|
+
) -> int:
|
|
107
|
+
"""Enqueue models waiting on ``upstream`` after it reaches ``model.done``.
|
|
108
|
+
|
|
109
|
+
``building`` is the set already in this apply, so a model the plan is
|
|
110
|
+
constructing is not queued again. One stamp shared by the waiters of a
|
|
111
|
+
single finish: a retry of the same finish dedupes, a later finish does not.
|
|
112
|
+
Returns how many runs were newly enqueued.
|
|
113
|
+
"""
|
|
114
|
+
table = after_waiters(project) if waiters is None else waiters
|
|
115
|
+
pending = [name for name in table.get(upstream, ()) if name not in building]
|
|
116
|
+
if not pending:
|
|
117
|
+
return 0
|
|
118
|
+
order = {model.name: index for index, model in enumerate(project.ordered())}
|
|
119
|
+
pending.sort(key=lambda name: order.get(name, 0))
|
|
120
|
+
token = stamp if stamp is not None else str(time.time_ns())
|
|
121
|
+
covered: set[str] = set(building)
|
|
122
|
+
enqueued = 0
|
|
123
|
+
for name in pending:
|
|
124
|
+
if name in covered:
|
|
125
|
+
continue
|
|
126
|
+
selector = scheduled_closure(project, [name])
|
|
127
|
+
key = f"after:{name}:{upstream}:{token}"
|
|
128
|
+
if await store.enqueue_run(key, selector, None, 0):
|
|
129
|
+
enqueued += 1
|
|
130
|
+
await store.append_event("run.enqueued", entity=key, payload={"models": selector, "after": upstream})
|
|
131
|
+
covered.update(selector)
|
|
132
|
+
return enqueued
|
|
@@ -22,6 +22,7 @@ from interlace.graph.project import CompiledModel, CompiledProject
|
|
|
22
22
|
from interlace.ir.canonicalize import table_references
|
|
23
23
|
from interlace.ir.layout import env_view
|
|
24
24
|
from interlace.ir.relation import TableRef
|
|
25
|
+
from interlace.scheduler.completion import after_waiters, schedule_kind, scheduled_closure
|
|
25
26
|
from interlace.scheduler.triggers import (
|
|
26
27
|
ABSENT,
|
|
27
28
|
CronTrigger,
|
|
@@ -34,21 +35,9 @@ from interlace.scheduler.triggers import (
|
|
|
34
35
|
from interlace.state.interval import Grain, as_grain, parse_grain
|
|
35
36
|
from interlace.state.store import SqliteStateStore
|
|
36
37
|
|
|
37
|
-
_SCHEDULE_KINDS = ("cron", "every", "watch", "on_change", "fresh", "webhook")
|
|
38
38
|
_IDENT = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
|
39
39
|
|
|
40
40
|
|
|
41
|
-
def schedule_kind(model: str, schedule: dict[str, str]) -> tuple[str, str]:
|
|
42
|
-
"""The one schedule key and its value. Cron, interval, file watch, table change, freshness, or webhook."""
|
|
43
|
-
present = [key for key in _SCHEDULE_KINDS if key in schedule]
|
|
44
|
-
if len(present) != 1:
|
|
45
|
-
raise DefinitionError(
|
|
46
|
-
f"model {model!r}: schedule needs exactly one of {', '.join(_SCHEDULE_KINDS)}",
|
|
47
|
-
details={"schedule": schedule},
|
|
48
|
-
)
|
|
49
|
-
return present[0], schedule[present[0]]
|
|
50
|
-
|
|
51
|
-
|
|
52
41
|
def webhook_targets(project: CompiledProject) -> dict[str, str]:
|
|
53
42
|
"""Map a webhook name to the model ``POST /hooks/{name}`` enqueues."""
|
|
54
43
|
found: dict[str, str] = {}
|
|
@@ -71,7 +60,8 @@ def build_triggers(project: CompiledProject, *, root: Path | None = None, enviro
|
|
|
71
60
|
``root`` is the project directory a ``watch:`` glob is relative to.
|
|
72
61
|
``environment`` names the view an ``on_change`` or ``fresh`` schedule reads when its
|
|
73
62
|
source is another model. Webhook schedules are validated here and enqueued
|
|
74
|
-
by ``POST /hooks/{name}
|
|
63
|
+
by ``POST /hooks/{name}``. ``after`` is validated here and enqueued when the
|
|
64
|
+
upstream model finishes. Neither becomes a ticking trigger.
|
|
75
65
|
"""
|
|
76
66
|
triggers: list[Trigger] = []
|
|
77
67
|
for model in project.models.values():
|
|
@@ -79,6 +69,7 @@ def build_triggers(project: CompiledProject, *, root: Path | None = None, enviro
|
|
|
79
69
|
if trigger is not None:
|
|
80
70
|
triggers.append(trigger)
|
|
81
71
|
webhook_targets(project) # reject two models sharing one hook name
|
|
72
|
+
after_waiters(project) # reject unknown names and cycles
|
|
82
73
|
return triggers
|
|
83
74
|
|
|
84
75
|
|
|
@@ -175,22 +166,6 @@ def _infer_change_table(
|
|
|
175
166
|
)
|
|
176
167
|
|
|
177
168
|
|
|
178
|
-
def scheduled_closure(project: CompiledProject, models: list[str]) -> list[str]:
|
|
179
|
-
"""The triggered models plus every model downstream of them.
|
|
180
|
-
|
|
181
|
-
A cron, interval, file watch, table change, freshness, or webhook means that model's inputs changed.
|
|
182
|
-
Downstream snapshots would otherwise keep the previous build. An explicit
|
|
183
|
-
``run --select`` is not expanded here: ``model``, ``model+``, and ``+model``
|
|
184
|
-
stay exactly what was asked for.
|
|
185
|
-
"""
|
|
186
|
-
chosen: set[str] = set()
|
|
187
|
-
for name in models:
|
|
188
|
-
chosen.add(name)
|
|
189
|
-
if name in project.models:
|
|
190
|
-
chosen |= project.graph.descendants(name)
|
|
191
|
-
return sorted(chosen)
|
|
192
|
-
|
|
193
|
-
|
|
194
169
|
class TriggerEngine:
|
|
195
170
|
"""Evaluates triggers on each tick and enqueues due runs."""
|
|
196
171
|
|
|
@@ -19,7 +19,7 @@ from litestar.static_files import create_static_files_router
|
|
|
19
19
|
from interlace import __version__
|
|
20
20
|
from interlace.exceptions import BreakingPlanError, LockError
|
|
21
21
|
from interlace.service.auth import auth_guard
|
|
22
|
-
from interlace.service.lifespan import project_lifespan
|
|
22
|
+
from interlace.service.lifespan import project_lifespan, startup_guard
|
|
23
23
|
from interlace.service.routes.admin import (
|
|
24
24
|
delete_apikey,
|
|
25
25
|
get_apikeys,
|
|
@@ -90,7 +90,8 @@ def create_app(
|
|
|
90
90
|
``scheduler=True`` makes this the combined daemon: the HTTP API plus a
|
|
91
91
|
background scheduler loop (tick triggers, drain the run queue) in one
|
|
92
92
|
process — the default for ``interlace serve``. ``apply_on_start`` applies the
|
|
93
|
-
project once
|
|
93
|
+
project once while the API is already accepting connections; writes get
|
|
94
|
+
503 until that apply finishes. The ``serve`` command turns it on.
|
|
94
95
|
``quack`` (a
|
|
95
96
|
``quack:<host>:<port>`` URI) additionally serves the warehouse over the
|
|
96
97
|
quack protocol so other processes — CLI runs, ad-hoc DuckDB clients —
|
|
@@ -220,7 +221,7 @@ def create_app(
|
|
|
220
221
|
stream_events,
|
|
221
222
|
],
|
|
222
223
|
lifespan=[lifespan],
|
|
223
|
-
guards=[auth_guard],
|
|
224
|
+
guards=[auth_guard, startup_guard],
|
|
224
225
|
openapi_config=OpenAPIConfig(
|
|
225
226
|
title="interlace",
|
|
226
227
|
version=__version__,
|
|
@@ -17,7 +17,9 @@ from pathlib import Path
|
|
|
17
17
|
from typing import cast
|
|
18
18
|
|
|
19
19
|
from litestar import Litestar
|
|
20
|
-
from litestar.
|
|
20
|
+
from litestar.connection import ASGIConnection
|
|
21
|
+
from litestar.exceptions import ImproperlyConfiguredException, ServiceUnavailableException
|
|
22
|
+
from litestar.handlers.base import BaseRouteHandler
|
|
21
23
|
|
|
22
24
|
from interlace.graph.column_lineage import column_lineage
|
|
23
25
|
from interlace.project import Project
|
|
@@ -38,6 +40,21 @@ from interlace.streaming.materializer import (
|
|
|
38
40
|
)
|
|
39
41
|
|
|
40
42
|
|
|
43
|
+
async def startup_guard(connection: ASGIConnection, _route_handler: BaseRouteHandler) -> None:
|
|
44
|
+
"""Refuse writes until the startup apply has finished.
|
|
45
|
+
|
|
46
|
+
The socket is open while that apply runs, so a client gets this response
|
|
47
|
+
instead of a refused connection. Reads stay available.
|
|
48
|
+
"""
|
|
49
|
+
status = getattr(connection.app.state, "startup_status", "ok")
|
|
50
|
+
if status == "ok" or connection.scope.get("method", "GET") in ("GET", "HEAD", "OPTIONS"):
|
|
51
|
+
return
|
|
52
|
+
if status == "starting":
|
|
53
|
+
raise ServiceUnavailableException(detail="startup apply is still running")
|
|
54
|
+
detail = getattr(connection.app.state, "startup_error", "") or "startup apply failed"
|
|
55
|
+
raise ServiceUnavailableException(detail=detail)
|
|
56
|
+
|
|
57
|
+
|
|
41
58
|
def project_lifespan( # noqa: C901
|
|
42
59
|
root: Path | str,
|
|
43
60
|
environment: str,
|
|
@@ -140,6 +157,10 @@ def project_lifespan( # noqa: C901
|
|
|
140
157
|
|
|
141
158
|
if streams:
|
|
142
159
|
app.state.flush_wanted.set() # catch up anything durable but unflushed at last shutdown
|
|
160
|
+
# "starting" until the first apply finishes. The socket opens at the yield
|
|
161
|
+
# below, so clients reach /health instead of a refused connection.
|
|
162
|
+
app.state.startup_status = "starting" if apply_on_start else "ok"
|
|
163
|
+
app.state.startup_error = ""
|
|
143
164
|
|
|
144
165
|
async def shutdown_watch() -> None:
|
|
145
166
|
"""End every open SSE stream the moment uvicorn begins shutting down.
|
|
@@ -159,16 +180,36 @@ def project_lifespan( # noqa: C901
|
|
|
159
180
|
with contextlib.suppress(asyncio.QueueFull):
|
|
160
181
|
subscriber.put_nowait(None)
|
|
161
182
|
|
|
162
|
-
|
|
163
|
-
|
|
183
|
+
flusher_task: asyncio.Task[None] | None = None
|
|
184
|
+
loop_task: asyncio.Task[None] | None = None
|
|
185
|
+
cdc_task: asyncio.Task[None] | None = None
|
|
186
|
+
|
|
187
|
+
async def open_service() -> None:
|
|
188
|
+
"""Apply first, then the loops. The listen socket is already open."""
|
|
189
|
+
nonlocal flusher_task, loop_task, cdc_task
|
|
190
|
+
try:
|
|
191
|
+
if apply_on_start:
|
|
192
|
+
logger.info("startup apply running; the API is accepting connections")
|
|
193
|
+
await startup_apply(app.state)
|
|
194
|
+
app.state.startup_status = "ok"
|
|
195
|
+
except Exception as exc:
|
|
196
|
+
app.state.startup_status = "error"
|
|
197
|
+
app.state.startup_error = str(exc)
|
|
198
|
+
logger.exception("startup apply failed")
|
|
199
|
+
server = getattr(app.state, "uvicorn_server", None)
|
|
200
|
+
if server is not None:
|
|
201
|
+
server.should_exit = True
|
|
202
|
+
return
|
|
203
|
+
if streams:
|
|
204
|
+
flusher_task = asyncio.create_task(flusher_loop(app.state, flush_interval=stream_flush_interval))
|
|
205
|
+
if scheduler:
|
|
206
|
+
loop_task = asyncio.create_task(scheduler_loop(app.state, interval=scheduler_interval))
|
|
207
|
+
if project.config.cdc:
|
|
208
|
+
cdc_task = asyncio.create_task(cdc_loop(app.state))
|
|
164
209
|
|
|
165
210
|
tail_task = asyncio.create_task(event_tail())
|
|
166
|
-
flusher_task = (
|
|
167
|
-
asyncio.create_task(flusher_loop(app.state, flush_interval=stream_flush_interval)) if streams else None
|
|
168
|
-
)
|
|
169
|
-
loop_task = asyncio.create_task(scheduler_loop(app.state, interval=scheduler_interval)) if scheduler else None
|
|
170
|
-
cdc_task = asyncio.create_task(cdc_loop(app.state)) if project.config.cdc else None
|
|
171
211
|
watch_task = asyncio.create_task(shutdown_watch())
|
|
212
|
+
service_task = asyncio.create_task(open_service())
|
|
172
213
|
try:
|
|
173
214
|
yield
|
|
174
215
|
finally:
|
|
@@ -183,6 +224,10 @@ def project_lifespan( # noqa: C901
|
|
|
183
224
|
for subscriber in list(app.state.sse_subscribers):
|
|
184
225
|
with contextlib.suppress(asyncio.QueueFull):
|
|
185
226
|
subscriber.put_nowait(None)
|
|
227
|
+
# Stop the opener first so it cannot start a loop after this snapshot.
|
|
228
|
+
service_task.cancel()
|
|
229
|
+
with contextlib.suppress(asyncio.CancelledError, Exception):
|
|
230
|
+
await service_task
|
|
186
231
|
for task in (cdc_task, loop_task, flusher_task, tail_task, watch_task):
|
|
187
232
|
if task is not None:
|
|
188
233
|
task.cancel()
|
|
@@ -142,6 +142,7 @@ async def get_schedules(state: State) -> list[ScheduleInfo]:
|
|
|
142
142
|
"watch": f"watch:{name}",
|
|
143
143
|
"on_change": f"change:{name}",
|
|
144
144
|
"fresh": f"fresh:{name}",
|
|
145
|
+
"after": f"after:{name}",
|
|
145
146
|
}.get(kind)
|
|
146
147
|
last = await state.store.get_trigger_last_fired(trigger_id) if trigger_id else None
|
|
147
148
|
|
|
@@ -33,7 +33,13 @@ from interlace.service.types import (
|
|
|
33
33
|
|
|
34
34
|
@get("/health")
|
|
35
35
|
async def health(state: State) -> dict[str, str]:
|
|
36
|
-
|
|
36
|
+
status = getattr(state, "startup_status", "ok")
|
|
37
|
+
body = {"status": status, "version": __version__, "environment": state.environment}
|
|
38
|
+
if status == "starting":
|
|
39
|
+
body["detail"] = "startup apply is still running"
|
|
40
|
+
elif status == "error":
|
|
41
|
+
body["detail"] = getattr(state, "startup_error", "") or "startup apply failed"
|
|
42
|
+
return body
|
|
37
43
|
|
|
38
44
|
|
|
39
45
|
@get("/", include_in_schema=False)
|
|
@@ -297,7 +297,7 @@ class ConnectionInfo(msgspec.Struct):
|
|
|
297
297
|
|
|
298
298
|
class ScheduleInfo(msgspec.Struct):
|
|
299
299
|
model: str
|
|
300
|
-
kind: str # "cron" | "every" | "watch" | "on_change" | "fresh" | "webhook"
|
|
300
|
+
kind: str # "cron" | "every" | "watch" | "on_change" | "fresh" | "after" | "webhook"
|
|
301
301
|
expression: str
|
|
302
302
|
next_fire: str | None
|
|
303
303
|
last_fired: str | None
|
|
@@ -143,7 +143,10 @@ export async function render(el, { api, toast, modal }) {
|
|
|
143
143
|
];
|
|
144
144
|
if (change.category) headBits.push(pill(change.category.replace("_", "-"), CATEGORY_TONE[change.category] ?? ""));
|
|
145
145
|
if (change.reused) headBits.push(pill("reused — no rebuild", "violet"));
|
|
146
|
-
if (change.impacted_columns.length)
|
|
146
|
+
if (change.impacted_columns.length) {
|
|
147
|
+
const mark = change.category === "non_breaking" ? "+" : "~";
|
|
148
|
+
headBits.push(h("span", { class: "sub" }, `${mark} ${change.impacted_columns.join(", ")}`));
|
|
149
|
+
}
|
|
147
150
|
|
|
148
151
|
const card = h("div", { class: "card" });
|
|
149
152
|
const detail = h("div", { class: "card-body" });
|
|
@@ -116,7 +116,7 @@ export async function render(el, { api, go, toast, modal, token }) {
|
|
|
116
116
|
schedules,
|
|
117
117
|
{
|
|
118
118
|
empty: "no scheduled models",
|
|
119
|
-
hint: "add schedule: {cron: …}, {every: …}, {watch: …}, {on_change: column}, {fresh: \"column 2h\"}, or {webhook: name}",
|
|
119
|
+
hint: "add schedule: {cron: …}, {every: …}, {watch: …}, {on_change: column}, {fresh: \"column 2h\"}, {after: model}, or {webhook: name}",
|
|
120
120
|
},
|
|
121
121
|
),
|
|
122
122
|
);
|