interlaced 2.8.4__tar.gz → 2.8.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {interlaced-2.8.4/src/interlaced.egg-info → interlaced-2.8.6}/PKG-INFO +1 -1
- {interlaced-2.8.4 → interlaced-2.8.6}/pyproject.toml +1 -1
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/config/config.py +8 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/dsl/decorators.py +10 -1
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/plan/differ.py +12 -1
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/plan/schedule.py +5 -1
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/project.py +8 -3
- interlaced-2.8.6/src/interlace/scheduler/__init__.py +39 -0
- interlaced-2.8.6/src/interlace/scheduler/completion.py +132 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/scheduler/engine.py +4 -29
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/service/routes/catalog.py +1 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/service/types.py +1 -1
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/service/ui/js/views/plan.js +4 -1
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/service/ui/js/views/system.js +1 -1
- interlaced-2.8.6/src/interlace/state/advisory.py +101 -0
- interlaced-2.8.6/src/interlace/state/conn.py +83 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/state/events.py +2 -2
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/state/locks.py +4 -4
- interlaced-2.8.6/src/interlace/state/pg.py +191 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/state/queue.py +8 -4
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/state/snapshots.py +3 -1
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/state/store.py +16 -9
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/streaming/log.py +33 -11
- {interlaced-2.8.4 → interlaced-2.8.6/src/interlaced.egg-info}/PKG-INFO +1 -1
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlaced.egg-info/SOURCES.txt +2 -0
- interlaced-2.8.4/src/interlace/scheduler/__init__.py +0 -17
- interlaced-2.8.4/src/interlace/state/advisory.py +0 -87
- interlaced-2.8.4/src/interlace/state/conn.py +0 -40
- {interlaced-2.8.4 → interlaced-2.8.6}/LICENSE +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/MANIFEST.in +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/README.md +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/setup.cfg +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/__init__.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/cdc/__init__.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/cdc/decode.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/cdc/publish.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/cdc/slot.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/checks/__init__.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/checks/builtin.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/checks/runner.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/checks/spec.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/cli/__init__.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/cli/build.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/cli/inspect.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/cli/main.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/cli/operate.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/cli/support.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/config/__init__.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/connections.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/contracts.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/dsl/__init__.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/dsl/discovery.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/dsl/dynamic.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/dsl/sql_config.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/engines/__init__.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/engines/adbc.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/engines/base.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/engines/bigquery.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/engines/duckdb.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/engines/postgres.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/engines/quack.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/engines/redshift.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/engines/registry.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/engines/snowflake.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/engines/spark.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/exceptions.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/graph/__init__.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/graph/column_lineage.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/graph/dag.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/graph/project.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/graph/selectors.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/inputs.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/inspect.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/ir/__init__.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/ir/canonicalize.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/ir/dialect.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/ir/fingerprint.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/ir/layout.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/ir/macros.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/ir/relation.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/ir/vars.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/mcp_server.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/physical/__init__.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/physical/changes.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/physical/ddl.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/physical/drift.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/physical/reconcile.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/physical/spec.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/plan/__init__.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/plan/annotate.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/plan/apply.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/plan/backfill.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/plan/comment.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/plan/delivery.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/plan/files.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/plan/fit.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/plan/orchestrate.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/plan/payload.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/plan/plan.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/plan/resolve.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/plan/result.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/plan/run.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/plan/table_diff.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/plan/transfer.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/py.typed +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/query.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/runtime/__init__.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/runtime/handles.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/runtime/python_model.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/scaffold.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/scheduler/daemon.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/scheduler/triggers.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/scheduler/worker.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/service/__init__.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/service/app.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/service/auth.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/service/lifespan.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/service/present.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/service/routes/__init__.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/service/routes/admin.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/service/routes/environments.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/service/routes/models.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/service/routes/plan.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/service/routes/runs.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/service/routes/streams.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/service/ui/app.css +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/service/ui/favicon.svg +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/service/ui/index.html +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/service/ui/js/api.js +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/service/ui/js/app.js +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/service/ui/js/dag.js +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/service/ui/js/timeline.js +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/service/ui/js/ui.js +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/service/ui/js/views/checks.js +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/service/ui/js/views/environments.js +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/service/ui/js/views/lineage.js +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/service/ui/js/views/models.js +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/service/ui/js/views/overview.js +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/service/ui/js/views/query.js +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/service/ui/js/views/runs.js +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/service/ui/js/views/streams.js +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/sinks.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/sources/__init__.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/sources/auth.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/sources/rest.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/state/__init__.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/state/beat.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/state/cdc.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/state/checks.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/state/codec.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/state/interval.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/state/janitor.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/state/keys.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/state/schema.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/state/snapshot.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/state/triggers.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/strategies/__init__.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/strategies/append.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/strategies/base.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/strategies/full_merge.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/strategies/hash_merge.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/strategies/incremental.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/strategies/merge.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/strategies/replace.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/strategies/replace_in_place.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/strategies/scd.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/strategies/view.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/streaming/__init__.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/streaming/materializer.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/streaming/schema.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/templates/cdc/README.md +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/templates/cdc/docker-compose.yml +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/templates/cdc/init/seed.sql +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/templates/cdc/interlace.yaml +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/templates/cdc/models/orders.sql +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/templates/cdc/models/orders_by_status.sql +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/templates/cdc/models/orders_stream.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/templates/cdc/push.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/templates/cdc/template.yaml +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/templates/events/README.md +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/templates/events/generate.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/templates/events/interlace.yaml +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/templates/events/models/events.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/templates/events/models/events_by_minute.sql +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/templates/events/models/events_by_type.sql +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/templates/events/models/top_users.sql +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/templates/events/models/user_spend.sql +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/templates/events/template.yaml +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/templates/github/README.md +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/templates/github/interlace.yaml +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/templates/github/models/github_issues.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/templates/github/models/issues_by_state.sql +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/templates/github/template.yaml +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/templates/postgres/README.md +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/templates/postgres/docker-compose.yml +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/templates/postgres/init/seed.sql +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/templates/postgres/interlace.yaml +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/templates/postgres/models/orders.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/templates/postgres/models/orders_by_status.sql +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/templates/postgres/template.yaml +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/templates/quickstart/README.md +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/templates/quickstart/interlace.yaml +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/templates/quickstart/models/enriched_events.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/templates/quickstart/models/event_summary.sql +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/templates/quickstart/models/raw_events.sql +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/templates/quickstart/template.yaml +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/testing/__init__.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlace/testing/golden.py +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlaced.egg-info/dependency_links.txt +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlaced.egg-info/entry_points.txt +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlaced.egg-info/requires.txt +0 -0
- {interlaced-2.8.4 → interlaced-2.8.6}/src/interlaced.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: interlaced
|
|
3
|
-
Version: 2.8.
|
|
3
|
+
Version: 2.8.6
|
|
4
4
|
Summary: Python and SQL models in one DAG. Transformation, orchestration and durable streaming in one process, on DuckDB and Postgres.
|
|
5
5
|
Author-email: Mark <mark@interlace.sh>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "interlaced"
|
|
7
|
-
version = "2.8.
|
|
7
|
+
version = "2.8.6"
|
|
8
8
|
description = "Python and SQL models in one DAG. Transformation, orchestration and durable streaming in one process, on DuckDB and Postgres."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.12"
|
|
@@ -250,6 +250,9 @@ class ProjectConfig(BaseModel):
|
|
|
250
250
|
default_engine: str = "default"
|
|
251
251
|
engines: dict[str, EngineConfig] = Field(default_factory=dict)
|
|
252
252
|
state_path: str = ".interlace/state.db" # SQLite control-plane database
|
|
253
|
+
# When set, snapshots, the queue, and the event log live in this Postgres
|
|
254
|
+
# database (schema ``interlace``) instead of ``state_path``. Not the warehouse.
|
|
255
|
+
state_url: str | None = None
|
|
253
256
|
# Optional NDJSON mirror of the operator event log. Unset writes nowhere.
|
|
254
257
|
# One JSON line is appended after each SQLite commit (CLI and daemon share it).
|
|
255
258
|
event_log_path: str | None = None
|
|
@@ -275,6 +278,11 @@ class ProjectConfig(BaseModel):
|
|
|
275
278
|
secrets: dict[str, SecretConfig] = Field(default_factory=dict)
|
|
276
279
|
quack_token: str | None = None # token for quack: databases (or INTERLACE_QUACK_TOKEN)
|
|
277
280
|
stream_path: str = ".interlace/streams.db" # durable stream log (SQLite WAL)
|
|
281
|
+
# When set, the stream log lives in this Postgres database (schema
|
|
282
|
+
# ``interlace_streams``) instead of ``stream_path``. A commit is durable
|
|
283
|
+
# under Postgres ``synchronous_commit`` (the default), the same promise as
|
|
284
|
+
# the SQLite WAL fsync.
|
|
285
|
+
stream_url: str | None = None
|
|
278
286
|
# Databases to ATTACH to the warehouse engine at open: alias -> DuckDB attach
|
|
279
287
|
# URI/path (a .duckdb file, "postgres:...", "sqlite:...", ...). Models can read
|
|
280
288
|
# them and table exports can write to them as <alias>.<schema>.<table>.
|
|
@@ -32,7 +32,8 @@ ModelFn = Callable[..., Any]
|
|
|
32
32
|
|
|
33
33
|
|
|
34
34
|
def normalize_schedule(schedule: Mapping[str, Any] | None, *, model: str) -> dict[str, str] | None:
|
|
35
|
-
"""Schedules are string values. ``fresh: {column, within}`` becomes ``"column within"``.
|
|
35
|
+
"""Schedules are string values. ``fresh: {column, within}`` becomes ``"column within"``.
|
|
36
|
+
``after: [a, b]`` becomes ``"a,b"``."""
|
|
36
37
|
if not schedule:
|
|
37
38
|
return None
|
|
38
39
|
normalised: dict[str, str] = {}
|
|
@@ -46,6 +47,14 @@ def normalize_schedule(schedule: Mapping[str, Any] | None, *, model: str) -> dic
|
|
|
46
47
|
raise DefinitionError(f"model {model!r}: fresh.within must be a window like '2h'")
|
|
47
48
|
normalised[key] = f"{column.strip()} {window.strip()}"
|
|
48
49
|
continue
|
|
50
|
+
if key == "after" and isinstance(value, (list, tuple)):
|
|
51
|
+
names = [str(item).strip() for item in value]
|
|
52
|
+
if not names or any(not name for name in names):
|
|
53
|
+
raise DefinitionError(f"model {model!r}: after must name a model, or a list of models")
|
|
54
|
+
normalised[key] = ",".join(names)
|
|
55
|
+
continue
|
|
56
|
+
if key == "after" and not isinstance(value, str):
|
|
57
|
+
raise DefinitionError(f"model {model!r}: after must name a model, or a list of models")
|
|
49
58
|
if not isinstance(value, str):
|
|
50
59
|
raise DefinitionError(f"model {model!r}: schedule {key!r} must be a string")
|
|
51
60
|
normalised[key] = value
|
|
@@ -338,6 +338,8 @@ async def diff( # noqa: C901
|
|
|
338
338
|
plan = Plan(environment=environment)
|
|
339
339
|
impact: dict[str, str] = {} # changed models only: "semantic" | "additive" | "clean"
|
|
340
340
|
touched: dict[str, frozenset[str] | None] = {} # semantic models: provably-changed columns (None = all)
|
|
341
|
+
# downstream models: the upstream columns that forced the rebuild
|
|
342
|
+
indirect: dict[str, frozenset[str]] = {}
|
|
341
343
|
|
|
342
344
|
previous_snapshots = await state.get_snapshots(
|
|
343
345
|
(name, fingerprint)
|
|
@@ -410,6 +412,8 @@ async def diff( # noqa: C901
|
|
|
410
412
|
consumed = _consumed_columns(model.ast, dep)
|
|
411
413
|
if dep_touched is not None and consumed is not None and not (consumed & dep_touched):
|
|
412
414
|
continue # column-pruned: reads only provably-unchanged columns
|
|
415
|
+
if dep_touched is not None and consumed is not None:
|
|
416
|
+
indirect[model.name] = consumed & dep_touched
|
|
413
417
|
verdict = "semantic"
|
|
414
418
|
break
|
|
415
419
|
if dep_impact == "additive" and _selects_star(model.ast):
|
|
@@ -436,8 +440,15 @@ async def diff( # noqa: C901
|
|
|
436
440
|
)
|
|
437
441
|
if inherit:
|
|
438
442
|
category = ChangeCategory.FORWARD_ONLY
|
|
443
|
+
shown = added
|
|
444
|
+
if impact.get(model.name) == "semantic":
|
|
445
|
+
own = touched.get(model.name)
|
|
446
|
+
if own:
|
|
447
|
+
shown = tuple(sorted(own))
|
|
448
|
+
elif model.name in indirect:
|
|
449
|
+
shown = tuple(sorted(indirect[model.name]))
|
|
439
450
|
plan.changes.append(
|
|
440
|
-
ModelChange(model.name, ChangeType.MODIFIED, category, previous_fingerprint, model.fingerprint,
|
|
451
|
+
ModelChange(model.name, ChangeType.MODIFIED, category, previous_fingerprint, model.fingerprint, shown)
|
|
441
452
|
)
|
|
442
453
|
if inherit: # copy-on-write: history seeds the NEW table; checks gate before views move
|
|
443
454
|
snapshot = replace(
|
|
@@ -25,7 +25,8 @@ from interlace.ir.relation import TableRef
|
|
|
25
25
|
from interlace.plan.backfill import run_backfill
|
|
26
26
|
from interlace.plan.plan import BackfillTask, Plan
|
|
27
27
|
from interlace.plan.result import ApplyResult, ProgressCallback, build_detail, failure_detail
|
|
28
|
-
from interlace.
|
|
28
|
+
from interlace.scheduler.completion import after_waiters, wake_after
|
|
29
|
+
from interlace.state.store import SqliteStateStore, StateStore
|
|
29
30
|
|
|
30
31
|
logger = logging.getLogger("interlace.apply")
|
|
31
32
|
|
|
@@ -123,6 +124,7 @@ async def schedule_builds( # noqa: C901
|
|
|
123
124
|
deps.add(ref)
|
|
124
125
|
blocking[name] = deps
|
|
125
126
|
|
|
127
|
+
waiting = after_waiters(compiled)
|
|
126
128
|
finished = {name: asyncio.Event() for name in per_model}
|
|
127
129
|
build_slots = asyncio.Semaphore(max(1, parallelism))
|
|
128
130
|
|
|
@@ -163,6 +165,8 @@ async def schedule_builds( # noqa: C901
|
|
|
163
165
|
finished[name].set()
|
|
164
166
|
if on_progress is not None:
|
|
165
167
|
on_progress(name, "done", build_detail(result, name))
|
|
168
|
+
if isinstance(state, SqliteStateStore):
|
|
169
|
+
await wake_after(state, compiled, name, building=set(per_model), waiters=waiting)
|
|
166
170
|
|
|
167
171
|
# Copied into each build task at creation, so a Python model that registers
|
|
168
172
|
# models (including from a worker thread) appends to this same list.
|
|
@@ -375,15 +375,20 @@ class Project:
|
|
|
375
375
|
)
|
|
376
376
|
|
|
377
377
|
async def open_state(self) -> SqliteStateStore:
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
378
|
+
if self.config.state_url:
|
|
379
|
+
store = await SqliteStateStore.open_postgres(self.config.state_url)
|
|
380
|
+
else:
|
|
381
|
+
path = self.root / self.config.state_path
|
|
382
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
383
|
+
store = await SqliteStateStore.open(path)
|
|
381
384
|
if self.config.event_log_path:
|
|
382
385
|
target = Path(self.config.event_log_path)
|
|
383
386
|
store.event_log_path = str(target if target.is_absolute() else self.root / target)
|
|
384
387
|
return store
|
|
385
388
|
|
|
386
389
|
async def open_stream_log(self) -> SqliteStreamLog:
|
|
390
|
+
if self.config.stream_url:
|
|
391
|
+
return await SqliteStreamLog.open_postgres(self.config.stream_url)
|
|
387
392
|
path = self.root / self.config.stream_path
|
|
388
393
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
389
394
|
return await SqliteStreamLog.open(path)
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"""Orchestration: triggers, the trigger engine, and the worker (queue = state store).
|
|
2
|
+
|
|
3
|
+
Submodules import each other. This package does not load them up front: importing
|
|
4
|
+
``interlace.scheduler.completion`` must not pull in the worker, which imports the
|
|
5
|
+
apply path that imports completion.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import importlib
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
__all__ = [
|
|
14
|
+
"CronTrigger",
|
|
15
|
+
"IntervalTrigger",
|
|
16
|
+
"RunRequest",
|
|
17
|
+
"Trigger",
|
|
18
|
+
"TriggerEngine",
|
|
19
|
+
"build_triggers",
|
|
20
|
+
"drain",
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
_EXPORTS: dict[str, tuple[str, str]] = {
|
|
24
|
+
"CronTrigger": ("interlace.scheduler.triggers", "CronTrigger"),
|
|
25
|
+
"IntervalTrigger": ("interlace.scheduler.triggers", "IntervalTrigger"),
|
|
26
|
+
"RunRequest": ("interlace.scheduler.triggers", "RunRequest"),
|
|
27
|
+
"Trigger": ("interlace.scheduler.triggers", "Trigger"),
|
|
28
|
+
"TriggerEngine": ("interlace.scheduler.engine", "TriggerEngine"),
|
|
29
|
+
"build_triggers": ("interlace.scheduler.engine", "build_triggers"),
|
|
30
|
+
"drain": ("interlace.scheduler.worker", "drain"),
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def __getattr__(name: str) -> Any:
|
|
35
|
+
found = _EXPORTS.get(name)
|
|
36
|
+
if found is None:
|
|
37
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
38
|
+
module_name, attr = found
|
|
39
|
+
return getattr(importlib.import_module(module_name), attr)
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
"""Run a model when another model finishes.
|
|
2
|
+
|
|
3
|
+
``schedule: {after: …}`` is not a clock tick. Apply calls :func:`wake_after` once
|
|
4
|
+
a build reaches ``model.done``. The trigger engine only validates the graph.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import time
|
|
10
|
+
|
|
11
|
+
from interlace.exceptions import DefinitionError
|
|
12
|
+
from interlace.graph.project import CompiledProject
|
|
13
|
+
from interlace.state.store import SqliteStateStore
|
|
14
|
+
|
|
15
|
+
_SCHEDULE_KINDS = ("cron", "every", "watch", "on_change", "fresh", "after", "webhook")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def schedule_kind(model: str, schedule: dict[str, str]) -> tuple[str, str]:
|
|
19
|
+
"""The one schedule key and its value.
|
|
20
|
+
|
|
21
|
+
Cron, interval, file watch, table change, freshness, upstream completion, or webhook.
|
|
22
|
+
"""
|
|
23
|
+
present = [key for key in _SCHEDULE_KINDS if key in schedule]
|
|
24
|
+
if len(present) != 1:
|
|
25
|
+
raise DefinitionError(
|
|
26
|
+
f"model {model!r}: schedule needs exactly one of {', '.join(_SCHEDULE_KINDS)}",
|
|
27
|
+
details={"schedule": schedule},
|
|
28
|
+
)
|
|
29
|
+
return present[0], schedule[present[0]]
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def scheduled_closure(project: CompiledProject, models: list[str]) -> list[str]:
|
|
33
|
+
"""The triggered models plus every model downstream of them.
|
|
34
|
+
|
|
35
|
+
A cron, interval, file watch, table change, freshness, upstream completion, or webhook
|
|
36
|
+
means that model's inputs changed. Downstream snapshots would otherwise keep the
|
|
37
|
+
previous build. An explicit ``run --select`` is not expanded here: ``model``,
|
|
38
|
+
``model+``, and ``+model`` stay exactly what was asked for.
|
|
39
|
+
"""
|
|
40
|
+
chosen: set[str] = set()
|
|
41
|
+
for name in models:
|
|
42
|
+
chosen.add(name)
|
|
43
|
+
if name in project.models:
|
|
44
|
+
chosen |= project.graph.descendants(name)
|
|
45
|
+
return sorted(chosen)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def after_waiters(project: CompiledProject) -> dict[str, list[str]]:
|
|
49
|
+
"""Map an upstream model to the models that run when it finishes.
|
|
50
|
+
|
|
51
|
+
``schedule: {after: raw}`` or ``{after: [raw, staging]}`` (stored as
|
|
52
|
+
``"raw,staging"``). A model cannot name itself, and the ``after`` edges
|
|
53
|
+
cannot cycle — otherwise two models would enqueue each other forever.
|
|
54
|
+
"""
|
|
55
|
+
found: dict[str, list[str]] = {}
|
|
56
|
+
for model in project.models.values():
|
|
57
|
+
if not model.schedule or "after" not in model.schedule:
|
|
58
|
+
continue
|
|
59
|
+
kind, value = schedule_kind(model.name, model.schedule)
|
|
60
|
+
if kind != "after":
|
|
61
|
+
continue
|
|
62
|
+
names = list(dict.fromkeys(part.strip() for part in value.split(",") if part.strip()))
|
|
63
|
+
if not names:
|
|
64
|
+
raise DefinitionError(f"model {model.name!r}: after must name a model")
|
|
65
|
+
if model.name in names:
|
|
66
|
+
raise DefinitionError(f"model {model.name!r}: after cannot name itself")
|
|
67
|
+
missing = [name for name in names if name not in project.models]
|
|
68
|
+
if missing:
|
|
69
|
+
raise DefinitionError(
|
|
70
|
+
f"model {model.name!r}: after names unknown models: {', '.join(missing)}",
|
|
71
|
+
details={"missing": missing},
|
|
72
|
+
)
|
|
73
|
+
for name in names:
|
|
74
|
+
found.setdefault(name, []).append(model.name)
|
|
75
|
+
_reject_after_cycles(found)
|
|
76
|
+
return found
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _reject_after_cycles(waiters: dict[str, list[str]]) -> None:
|
|
80
|
+
visiting: set[str] = set()
|
|
81
|
+
visited: set[str] = set()
|
|
82
|
+
|
|
83
|
+
def walk(node: str) -> None:
|
|
84
|
+
if node in visited:
|
|
85
|
+
return
|
|
86
|
+
if node in visiting:
|
|
87
|
+
raise DefinitionError(f"schedule after has a cycle through {node!r}")
|
|
88
|
+
visiting.add(node)
|
|
89
|
+
for nxt in waiters.get(node, ()):
|
|
90
|
+
walk(nxt)
|
|
91
|
+
visiting.remove(node)
|
|
92
|
+
visited.add(node)
|
|
93
|
+
|
|
94
|
+
for node in list(waiters):
|
|
95
|
+
walk(node)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
async def wake_after(
|
|
99
|
+
store: SqliteStateStore,
|
|
100
|
+
project: CompiledProject,
|
|
101
|
+
upstream: str,
|
|
102
|
+
*,
|
|
103
|
+
building: set[str],
|
|
104
|
+
waiters: dict[str, list[str]] | None = None,
|
|
105
|
+
stamp: str | None = None,
|
|
106
|
+
) -> int:
|
|
107
|
+
"""Enqueue models waiting on ``upstream`` after it reaches ``model.done``.
|
|
108
|
+
|
|
109
|
+
``building`` is the set already in this apply, so a model the plan is
|
|
110
|
+
constructing is not queued again. One stamp shared by the waiters of a
|
|
111
|
+
single finish: a retry of the same finish dedupes, a later finish does not.
|
|
112
|
+
Returns how many runs were newly enqueued.
|
|
113
|
+
"""
|
|
114
|
+
table = after_waiters(project) if waiters is None else waiters
|
|
115
|
+
pending = [name for name in table.get(upstream, ()) if name not in building]
|
|
116
|
+
if not pending:
|
|
117
|
+
return 0
|
|
118
|
+
order = {model.name: index for index, model in enumerate(project.ordered())}
|
|
119
|
+
pending.sort(key=lambda name: order.get(name, 0))
|
|
120
|
+
token = stamp if stamp is not None else str(time.time_ns())
|
|
121
|
+
covered: set[str] = set(building)
|
|
122
|
+
enqueued = 0
|
|
123
|
+
for name in pending:
|
|
124
|
+
if name in covered:
|
|
125
|
+
continue
|
|
126
|
+
selector = scheduled_closure(project, [name])
|
|
127
|
+
key = f"after:{name}:{upstream}:{token}"
|
|
128
|
+
if await store.enqueue_run(key, selector, None, 0):
|
|
129
|
+
enqueued += 1
|
|
130
|
+
await store.append_event("run.enqueued", entity=key, payload={"models": selector, "after": upstream})
|
|
131
|
+
covered.update(selector)
|
|
132
|
+
return enqueued
|
|
@@ -22,6 +22,7 @@ from interlace.graph.project import CompiledModel, CompiledProject
|
|
|
22
22
|
from interlace.ir.canonicalize import table_references
|
|
23
23
|
from interlace.ir.layout import env_view
|
|
24
24
|
from interlace.ir.relation import TableRef
|
|
25
|
+
from interlace.scheduler.completion import after_waiters, schedule_kind, scheduled_closure
|
|
25
26
|
from interlace.scheduler.triggers import (
|
|
26
27
|
ABSENT,
|
|
27
28
|
CronTrigger,
|
|
@@ -34,21 +35,9 @@ from interlace.scheduler.triggers import (
|
|
|
34
35
|
from interlace.state.interval import Grain, as_grain, parse_grain
|
|
35
36
|
from interlace.state.store import SqliteStateStore
|
|
36
37
|
|
|
37
|
-
_SCHEDULE_KINDS = ("cron", "every", "watch", "on_change", "fresh", "webhook")
|
|
38
38
|
_IDENT = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
|
39
39
|
|
|
40
40
|
|
|
41
|
-
def schedule_kind(model: str, schedule: dict[str, str]) -> tuple[str, str]:
|
|
42
|
-
"""The one schedule key and its value. Cron, interval, file watch, table change, freshness, or webhook."""
|
|
43
|
-
present = [key for key in _SCHEDULE_KINDS if key in schedule]
|
|
44
|
-
if len(present) != 1:
|
|
45
|
-
raise DefinitionError(
|
|
46
|
-
f"model {model!r}: schedule needs exactly one of {', '.join(_SCHEDULE_KINDS)}",
|
|
47
|
-
details={"schedule": schedule},
|
|
48
|
-
)
|
|
49
|
-
return present[0], schedule[present[0]]
|
|
50
|
-
|
|
51
|
-
|
|
52
41
|
def webhook_targets(project: CompiledProject) -> dict[str, str]:
|
|
53
42
|
"""Map a webhook name to the model ``POST /hooks/{name}`` enqueues."""
|
|
54
43
|
found: dict[str, str] = {}
|
|
@@ -71,7 +60,8 @@ def build_triggers(project: CompiledProject, *, root: Path | None = None, enviro
|
|
|
71
60
|
``root`` is the project directory a ``watch:`` glob is relative to.
|
|
72
61
|
``environment`` names the view an ``on_change`` or ``fresh`` schedule reads when its
|
|
73
62
|
source is another model. Webhook schedules are validated here and enqueued
|
|
74
|
-
by ``POST /hooks/{name}
|
|
63
|
+
by ``POST /hooks/{name}``. ``after`` is validated here and enqueued when the
|
|
64
|
+
upstream model finishes. Neither becomes a ticking trigger.
|
|
75
65
|
"""
|
|
76
66
|
triggers: list[Trigger] = []
|
|
77
67
|
for model in project.models.values():
|
|
@@ -79,6 +69,7 @@ def build_triggers(project: CompiledProject, *, root: Path | None = None, enviro
|
|
|
79
69
|
if trigger is not None:
|
|
80
70
|
triggers.append(trigger)
|
|
81
71
|
webhook_targets(project) # reject two models sharing one hook name
|
|
72
|
+
after_waiters(project) # reject unknown names and cycles
|
|
82
73
|
return triggers
|
|
83
74
|
|
|
84
75
|
|
|
@@ -175,22 +166,6 @@ def _infer_change_table(
|
|
|
175
166
|
)
|
|
176
167
|
|
|
177
168
|
|
|
178
|
-
def scheduled_closure(project: CompiledProject, models: list[str]) -> list[str]:
|
|
179
|
-
"""The triggered models plus every model downstream of them.
|
|
180
|
-
|
|
181
|
-
A cron, interval, file watch, table change, freshness, or webhook means that model's inputs changed.
|
|
182
|
-
Downstream snapshots would otherwise keep the previous build. An explicit
|
|
183
|
-
``run --select`` is not expanded here: ``model``, ``model+``, and ``+model``
|
|
184
|
-
stay exactly what was asked for.
|
|
185
|
-
"""
|
|
186
|
-
chosen: set[str] = set()
|
|
187
|
-
for name in models:
|
|
188
|
-
chosen.add(name)
|
|
189
|
-
if name in project.models:
|
|
190
|
-
chosen |= project.graph.descendants(name)
|
|
191
|
-
return sorted(chosen)
|
|
192
|
-
|
|
193
|
-
|
|
194
169
|
class TriggerEngine:
|
|
195
170
|
"""Evaluates triggers on each tick and enqueues due runs."""
|
|
196
171
|
|
|
@@ -142,6 +142,7 @@ async def get_schedules(state: State) -> list[ScheduleInfo]:
|
|
|
142
142
|
"watch": f"watch:{name}",
|
|
143
143
|
"on_change": f"change:{name}",
|
|
144
144
|
"fresh": f"fresh:{name}",
|
|
145
|
+
"after": f"after:{name}",
|
|
145
146
|
}.get(kind)
|
|
146
147
|
last = await state.store.get_trigger_last_fired(trigger_id) if trigger_id else None
|
|
147
148
|
|
|
@@ -297,7 +297,7 @@ class ConnectionInfo(msgspec.Struct):
|
|
|
297
297
|
|
|
298
298
|
class ScheduleInfo(msgspec.Struct):
|
|
299
299
|
model: str
|
|
300
|
-
kind: str # "cron" | "every" | "watch" | "on_change" | "fresh" | "webhook"
|
|
300
|
+
kind: str # "cron" | "every" | "watch" | "on_change" | "fresh" | "after" | "webhook"
|
|
301
301
|
expression: str
|
|
302
302
|
next_fire: str | None
|
|
303
303
|
last_fired: str | None
|
|
@@ -143,7 +143,10 @@ export async function render(el, { api, toast, modal }) {
|
|
|
143
143
|
];
|
|
144
144
|
if (change.category) headBits.push(pill(change.category.replace("_", "-"), CATEGORY_TONE[change.category] ?? ""));
|
|
145
145
|
if (change.reused) headBits.push(pill("reused — no rebuild", "violet"));
|
|
146
|
-
if (change.impacted_columns.length)
|
|
146
|
+
if (change.impacted_columns.length) {
|
|
147
|
+
const mark = change.category === "non_breaking" ? "+" : "~";
|
|
148
|
+
headBits.push(h("span", { class: "sub" }, `${mark} ${change.impacted_columns.join(", ")}`));
|
|
149
|
+
}
|
|
147
150
|
|
|
148
151
|
const card = h("div", { class: "card" });
|
|
149
152
|
const detail = h("div", { class: "card-body" });
|
|
@@ -116,7 +116,7 @@ export async function render(el, { api, go, toast, modal, token }) {
|
|
|
116
116
|
schedules,
|
|
117
117
|
{
|
|
118
118
|
empty: "no scheduled models",
|
|
119
|
-
hint: "add schedule: {cron: …}, {every: …}, {watch: …}, {on_change: column}, {fresh: \"column 2h\"}, or {webhook: name}",
|
|
119
|
+
hint: "add schedule: {cron: …}, {every: …}, {watch: …}, {on_change: column}, {fresh: \"column 2h\"}, {after: model}, or {webhook: name}",
|
|
120
120
|
},
|
|
121
121
|
),
|
|
122
122
|
);
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
"""Cross-process advisory locks stored in the control-plane database.
|
|
2
|
+
|
|
3
|
+
SQLite takes the lock inside ``BEGIN IMMEDIATE``, which serialises writers.
|
|
4
|
+
Postgres does not, so the same decision is one conditional upsert: the row
|
|
5
|
+
changes only when this owner already holds it or the lease has expired.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import time as _time
|
|
11
|
+
from datetime import UTC, datetime, timedelta
|
|
12
|
+
|
|
13
|
+
from interlace.state.conn import ControlDb
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class AdvisoryLockStore:
|
|
17
|
+
"""Advisory locks on one control-plane connection."""
|
|
18
|
+
|
|
19
|
+
def __init__(self, db: ControlDb) -> None:
|
|
20
|
+
self._db = db
|
|
21
|
+
|
|
22
|
+
async def acquire_lock(self, name: str, *, owner: str, lease_seconds: float = 180.0, timeout: float = 60.0) -> bool:
|
|
23
|
+
"""Take (or renew, if already owned) a named advisory lock. Returns False on timeout."""
|
|
24
|
+
return await self._db.io(self._acquire_lock_sync, name, owner, lease_seconds, timeout)
|
|
25
|
+
|
|
26
|
+
def _acquire_lock_sync(self, name: str, owner: str, lease_seconds: float, timeout: float) -> bool:
|
|
27
|
+
deadline = _time.monotonic() + max(0.0, timeout)
|
|
28
|
+
while True:
|
|
29
|
+
if self._try_acquire(name, owner, lease_seconds):
|
|
30
|
+
return True
|
|
31
|
+
if _time.monotonic() >= deadline:
|
|
32
|
+
return False
|
|
33
|
+
_time.sleep(0.05)
|
|
34
|
+
|
|
35
|
+
def _try_acquire(self, name: str, owner: str, lease_seconds: float) -> bool:
|
|
36
|
+
with self._db.lock:
|
|
37
|
+
if self._db.dialect == "postgres":
|
|
38
|
+
return self._try_acquire_postgres(name, owner, lease_seconds)
|
|
39
|
+
return self._try_acquire_sqlite(name, owner, lease_seconds)
|
|
40
|
+
|
|
41
|
+
def _try_acquire_sqlite(self, name: str, owner: str, lease_seconds: float) -> bool:
|
|
42
|
+
self._db.conn.execute("BEGIN IMMEDIATE")
|
|
43
|
+
try:
|
|
44
|
+
now = datetime.now(UTC)
|
|
45
|
+
row = self._db.conn.execute(
|
|
46
|
+
"SELECT owner, expires_at FROM advisory_locks WHERE name = ?", (name,)
|
|
47
|
+
).fetchone()
|
|
48
|
+
held = row is not None and datetime.fromisoformat(row["expires_at"]) > now
|
|
49
|
+
if held and row["owner"] != owner:
|
|
50
|
+
self._db.conn.commit()
|
|
51
|
+
return False
|
|
52
|
+
expires = (now + timedelta(seconds=lease_seconds)).isoformat()
|
|
53
|
+
self._db.conn.execute(
|
|
54
|
+
"INSERT INTO advisory_locks (name, owner, expires_at) VALUES (?, ?, ?) "
|
|
55
|
+
"ON CONFLICT(name) DO UPDATE SET owner = excluded.owner, expires_at = excluded.expires_at",
|
|
56
|
+
(name, owner, expires),
|
|
57
|
+
)
|
|
58
|
+
self._db.conn.commit()
|
|
59
|
+
return True
|
|
60
|
+
except BaseException:
|
|
61
|
+
self._db.conn.rollback()
|
|
62
|
+
raise
|
|
63
|
+
|
|
64
|
+
def _try_acquire_postgres(self, name: str, owner: str, lease_seconds: float) -> bool:
|
|
65
|
+
now = datetime.now(UTC)
|
|
66
|
+
expires = (now + timedelta(seconds=lease_seconds)).isoformat()
|
|
67
|
+
# One statement. A lost UPDATE matches nothing, so two sessions cannot both observe a win.
|
|
68
|
+
# Compare the lease as timestamptz: the stored text is ISO-8601, which does not sort as text.
|
|
69
|
+
row = self._db.conn.execute(
|
|
70
|
+
"INSERT INTO advisory_locks (name, owner, expires_at) VALUES (?, ?, ?) "
|
|
71
|
+
"ON CONFLICT(name) DO UPDATE SET owner = excluded.owner, expires_at = excluded.expires_at "
|
|
72
|
+
"WHERE advisory_locks.owner = excluded.owner "
|
|
73
|
+
"OR advisory_locks.expires_at::timestamptz <= ?::timestamptz "
|
|
74
|
+
"RETURNING owner",
|
|
75
|
+
(name, owner, expires, now.isoformat()),
|
|
76
|
+
).fetchone()
|
|
77
|
+
return row is not None and row["owner"] == owner
|
|
78
|
+
|
|
79
|
+
async def renew_lock(self, name: str, *, owner: str, lease_seconds: float = 180.0) -> bool:
|
|
80
|
+
"""Extend a lock only if ``owner`` still holds it. Returns False if lost."""
|
|
81
|
+
return await self._db.io(self.renew_lock_sync, name, owner, lease_seconds)
|
|
82
|
+
|
|
83
|
+
def renew_lock_sync(self, name: str, owner: str, lease_seconds: float) -> bool:
|
|
84
|
+
"""Extend the lock off the event loop. False when ``owner`` no longer holds it."""
|
|
85
|
+
expires = (datetime.now(UTC) + timedelta(seconds=lease_seconds)).isoformat()
|
|
86
|
+
with self._db.lock:
|
|
87
|
+
cursor = self._db.conn.execute(
|
|
88
|
+
"UPDATE advisory_locks SET expires_at = ? WHERE name = ? AND owner = ?",
|
|
89
|
+
(expires, name, owner),
|
|
90
|
+
)
|
|
91
|
+
self._db.conn.commit()
|
|
92
|
+
return cursor.rowcount > 0
|
|
93
|
+
|
|
94
|
+
async def release_lock(self, name: str, *, owner: str) -> None:
|
|
95
|
+
"""Drop a lock if ``owner`` still holds it (no-op otherwise)."""
|
|
96
|
+
await self._db.io(self._release_lock_sync, name, owner)
|
|
97
|
+
|
|
98
|
+
def _release_lock_sync(self, name: str, owner: str) -> None:
|
|
99
|
+
with self._db.lock:
|
|
100
|
+
self._db.conn.execute("DELETE FROM advisory_locks WHERE name = ? AND owner = ?", (name, owner))
|
|
101
|
+
self._db.conn.commit()
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
"""One connection shared by every control-plane store.
|
|
2
|
+
|
|
3
|
+
SQLite is the default. ``connect_postgres`` opens the same stores on Postgres.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import asyncio
|
|
9
|
+
import sqlite3
|
|
10
|
+
import threading
|
|
11
|
+
from collections.abc import Callable, Sequence
|
|
12
|
+
from typing import Any, Protocol, TypeVar
|
|
13
|
+
|
|
14
|
+
from interlace.state.schema import _migrate
|
|
15
|
+
|
|
16
|
+
_R = TypeVar("_R")
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class _Cursor(Protocol):
|
|
20
|
+
lastrowid: int
|
|
21
|
+
rowcount: int
|
|
22
|
+
|
|
23
|
+
def fetchone(self) -> Any: ...
|
|
24
|
+
|
|
25
|
+
def fetchall(self) -> Sequence[Any]: ...
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class ConnLike(Protocol):
|
|
29
|
+
"""The slice of ``sqlite3.Connection`` the stores use. Postgres implements it too."""
|
|
30
|
+
|
|
31
|
+
def execute(self, sql: str, parameters: Any = ..., /) -> _Cursor: ...
|
|
32
|
+
|
|
33
|
+
def executemany(self, sql: str, seq_of_parameters: Any, /) -> _Cursor: ...
|
|
34
|
+
|
|
35
|
+
def commit(self) -> None: ...
|
|
36
|
+
|
|
37
|
+
def rollback(self) -> None: ...
|
|
38
|
+
|
|
39
|
+
def close(self) -> None: ...
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class ControlDb:
|
|
43
|
+
"""The single control-plane connection. Stores take this; they do not open their own.
|
|
44
|
+
|
|
45
|
+
The default is a SQLite file. ``connect_postgres`` is the same stores on a
|
|
46
|
+
Postgres schema (``state_url``).
|
|
47
|
+
"""
|
|
48
|
+
|
|
49
|
+
def __init__(self, connection: ConnLike, *, dialect: str = "sqlite") -> None:
|
|
50
|
+
self.conn = connection
|
|
51
|
+
self.lock = threading.Lock()
|
|
52
|
+
self.dialect = dialect
|
|
53
|
+
|
|
54
|
+
@staticmethod
|
|
55
|
+
def connect(path: str) -> sqlite3.Connection:
|
|
56
|
+
conn = sqlite3.connect(path, check_same_thread=False)
|
|
57
|
+
conn.row_factory = sqlite3.Row
|
|
58
|
+
conn.execute("PRAGMA journal_mode = WAL")
|
|
59
|
+
conn.execute("PRAGMA synchronous = NORMAL")
|
|
60
|
+
conn.execute("PRAGMA foreign_keys = ON")
|
|
61
|
+
conn.execute("PRAGMA busy_timeout = 5000") # CLI + daemon share this file: wait, don't error
|
|
62
|
+
conn.execute("PRAGMA cache_size = -65536")
|
|
63
|
+
_migrate(conn)
|
|
64
|
+
return conn
|
|
65
|
+
|
|
66
|
+
@staticmethod
|
|
67
|
+
def connect_postgres(dsn: str, schema: str = "interlace") -> ConnLike:
|
|
68
|
+
from interlace.state.pg import connect, migrate
|
|
69
|
+
|
|
70
|
+
conn = connect(dsn, schema)
|
|
71
|
+
try:
|
|
72
|
+
migrate(conn, schema)
|
|
73
|
+
except Exception:
|
|
74
|
+
conn.close()
|
|
75
|
+
raise
|
|
76
|
+
return conn
|
|
77
|
+
|
|
78
|
+
async def io(self, fn: Callable[..., _R], /, *args: Any, **kwargs: Any) -> _R:
|
|
79
|
+
"""The one worker-thread entry. Sync methods keep the connection lock."""
|
|
80
|
+
return await asyncio.to_thread(fn, *args, **kwargs)
|
|
81
|
+
|
|
82
|
+
async def close(self) -> None:
|
|
83
|
+
await self.io(self.conn.close)
|
|
@@ -49,8 +49,8 @@ class EventLogStore:
|
|
|
49
49
|
with self._db.lock:
|
|
50
50
|
rows = self._db.conn.execute(
|
|
51
51
|
"SELECT seq, ts, type, entity, payload FROM event_log "
|
|
52
|
-
"WHERE json_extract(payload, '$.run') = ? ORDER BY seq",
|
|
53
|
-
(run_id,),
|
|
52
|
+
"WHERE CAST(json_extract(payload, '$.run') AS TEXT) = ? ORDER BY seq",
|
|
53
|
+
(str(run_id),),
|
|
54
54
|
).fetchall()
|
|
55
55
|
return [self._event_row(row) for row in rows]
|
|
56
56
|
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
"""Cross-process advisory locks for warehouse-mutating work.
|
|
2
2
|
|
|
3
3
|
CLI ``apply`` / ``run`` and the daemon (HTTP apply, stream flush, scheduler drain,
|
|
4
|
-
gc, env drop/rollback)
|
|
5
|
-
``asyncio.Lock`` cannot
|
|
6
|
-
:meth:`AdvisoryLockStore.acquire_lock`
|
|
7
|
-
section runs.
|
|
4
|
+
gc, env drop/rollback) share one control plane: the SQLite state file, or the
|
|
5
|
+
Postgres schema from ``state_url``. An in-process ``asyncio.Lock`` cannot
|
|
6
|
+
serialise those writers — this module does, via :meth:`AdvisoryLockStore.acquire_lock`
|
|
7
|
+
with a heartbeat while the critical section runs.
|
|
8
8
|
"""
|
|
9
9
|
|
|
10
10
|
from __future__ import annotations
|