interlaced 2.8.2__tar.gz → 2.8.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {interlaced-2.8.2/src/interlaced.egg-info → interlaced-2.8.3}/PKG-INFO +6 -7
- {interlaced-2.8.2 → interlaced-2.8.3}/README.md +5 -6
- {interlaced-2.8.2 → interlaced-2.8.3}/pyproject.toml +3 -2
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/cli/build.py +2 -2
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/cli/operate.py +9 -1
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/cli/support.py +1 -1
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/config/config.py +71 -2
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/dsl/dynamic.py +1 -1
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/engines/adbc.py +6 -4
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/engines/base.py +20 -1
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/engines/duckdb.py +5 -3
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/engines/postgres.py +9 -5
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/engines/registry.py +4 -0
- interlaced-2.8.3/src/interlace/engines/snowflake.py +89 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/graph/project.py +34 -7
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/inputs.py +71 -0
- interlaced-2.8.3/src/interlace/ir/dialect.py +111 -0
- interlaced-2.8.3/src/interlace/ir/fingerprint.py +231 -0
- interlaced-2.8.3/src/interlace/ir/vars.py +88 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/mcp_server.py +1 -1
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/physical/drift.py +1 -1
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/physical/spec.py +1 -1
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/plan/annotate.py +1 -1
- interlaced-2.8.3/src/interlace/plan/apply.py +168 -0
- interlaced-2.8.3/src/interlace/plan/backfill.py +441 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/plan/comment.py +1 -1
- interlaced-2.8.3/src/interlace/plan/delivery.py +249 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/plan/differ.py +3 -3
- interlaced-2.8.3/src/interlace/plan/files.py +79 -0
- interlaced-2.8.3/src/interlace/plan/fit.py +234 -0
- interlaced-2.8.3/src/interlace/plan/result.py +68 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/plan/run.py +64 -36
- interlaced-2.8.3/src/interlace/plan/schedule.py +185 -0
- interlaced-2.8.3/src/interlace/plan/transfer.py +89 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/project.py +30 -9
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/runtime/python_model.py +3 -1
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/scheduler/daemon.py +83 -3
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/scheduler/engine.py +21 -5
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/scheduler/worker.py +49 -24
- interlaced-2.8.3/src/interlace/service/app.py +226 -0
- interlaced-2.8.2/src/interlace/service/app.py → interlaced-2.8.3/src/interlace/service/lifespan.py +22 -206
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/service/present.py +3 -2
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/service/routes/runs.py +4 -3
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/service/routes/streams.py +2 -2
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/sinks.py +39 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/state/advisory.py +6 -3
- interlaced-2.8.3/src/interlace/state/beat.py +43 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/state/janitor.py +2 -2
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/state/locks.py +9 -19
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/state/queue.py +3 -2
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/state/store.py +9 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/strategies/__init__.py +1 -1
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/strategies/full_merge.py +1 -1
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/streaming/materializer.py +9 -2
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/streaming/schema.py +1 -1
- {interlaced-2.8.2 → interlaced-2.8.3/src/interlaced.egg-info}/PKG-INFO +6 -7
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlaced.egg-info/SOURCES.txt +11 -0
- interlaced-2.8.2/src/interlace/engines/snowflake.py +0 -44
- interlaced-2.8.2/src/interlace/ir/fingerprint.py +0 -67
- interlaced-2.8.2/src/interlace/plan/apply.py +0 -1145
- {interlaced-2.8.2 → interlaced-2.8.3}/LICENSE +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/MANIFEST.in +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/setup.cfg +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/__init__.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/cdc/__init__.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/cdc/decode.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/cdc/publish.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/cdc/slot.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/checks/__init__.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/checks/builtin.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/checks/runner.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/checks/spec.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/cli/__init__.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/cli/inspect.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/cli/main.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/config/__init__.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/connections.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/contracts.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/dsl/__init__.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/dsl/decorators.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/dsl/discovery.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/dsl/sql_config.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/engines/__init__.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/engines/bigquery.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/engines/quack.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/engines/redshift.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/engines/spark.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/exceptions.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/graph/__init__.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/graph/column_lineage.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/graph/dag.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/graph/selectors.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/inspect.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/ir/__init__.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/ir/canonicalize.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/ir/layout.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/ir/macros.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/ir/relation.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/physical/__init__.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/physical/changes.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/physical/ddl.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/physical/reconcile.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/plan/__init__.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/plan/orchestrate.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/plan/payload.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/plan/plan.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/plan/resolve.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/plan/table_diff.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/py.typed +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/query.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/runtime/__init__.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/runtime/handles.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/scaffold.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/scheduler/__init__.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/scheduler/triggers.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/service/__init__.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/service/auth.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/service/routes/__init__.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/service/routes/admin.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/service/routes/catalog.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/service/routes/environments.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/service/routes/models.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/service/routes/plan.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/service/types.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/service/ui/app.css +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/service/ui/favicon.svg +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/service/ui/index.html +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/service/ui/js/api.js +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/service/ui/js/app.js +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/service/ui/js/dag.js +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/service/ui/js/timeline.js +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/service/ui/js/ui.js +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/service/ui/js/views/checks.js +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/service/ui/js/views/environments.js +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/service/ui/js/views/lineage.js +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/service/ui/js/views/models.js +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/service/ui/js/views/overview.js +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/service/ui/js/views/plan.js +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/service/ui/js/views/query.js +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/service/ui/js/views/runs.js +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/service/ui/js/views/streams.js +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/service/ui/js/views/system.js +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/sources/__init__.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/sources/auth.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/sources/rest.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/state/__init__.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/state/cdc.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/state/checks.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/state/codec.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/state/conn.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/state/events.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/state/interval.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/state/keys.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/state/schema.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/state/snapshot.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/state/snapshots.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/state/triggers.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/strategies/append.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/strategies/base.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/strategies/hash_merge.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/strategies/incremental.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/strategies/merge.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/strategies/replace.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/strategies/replace_in_place.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/strategies/scd.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/strategies/view.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/streaming/__init__.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/streaming/log.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/templates/cdc/README.md +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/templates/cdc/docker-compose.yml +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/templates/cdc/init/seed.sql +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/templates/cdc/interlace.yaml +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/templates/cdc/models/orders.sql +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/templates/cdc/models/orders_by_status.sql +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/templates/cdc/models/orders_stream.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/templates/cdc/push.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/templates/cdc/template.yaml +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/templates/events/README.md +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/templates/events/generate.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/templates/events/interlace.yaml +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/templates/events/models/events.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/templates/events/models/events_by_minute.sql +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/templates/events/models/events_by_type.sql +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/templates/events/models/top_users.sql +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/templates/events/models/user_spend.sql +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/templates/events/template.yaml +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/templates/github/README.md +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/templates/github/interlace.yaml +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/templates/github/models/github_issues.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/templates/github/models/issues_by_state.sql +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/templates/github/template.yaml +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/templates/postgres/README.md +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/templates/postgres/docker-compose.yml +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/templates/postgres/init/seed.sql +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/templates/postgres/interlace.yaml +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/templates/postgres/models/orders.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/templates/postgres/models/orders_by_status.sql +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/templates/postgres/template.yaml +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/templates/quickstart/README.md +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/templates/quickstart/interlace.yaml +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/templates/quickstart/models/enriched_events.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/templates/quickstart/models/event_summary.sql +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/templates/quickstart/models/raw_events.sql +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/templates/quickstart/template.yaml +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/testing/__init__.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlace/testing/golden.py +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlaced.egg-info/dependency_links.txt +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlaced.egg-info/entry_points.txt +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlaced.egg-info/requires.txt +0 -0
- {interlaced-2.8.2 → interlaced-2.8.3}/src/interlaced.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: interlaced
|
|
3
|
-
Version: 2.8.
|
|
3
|
+
Version: 2.8.3
|
|
4
4
|
Summary: Python and SQL models in one DAG. Transformation, orchestration and durable streaming in one process, on DuckDB and Postgres.
|
|
5
5
|
Author-email: Mark <mark@interlace.sh>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -110,12 +110,11 @@ pip install 'interlaced[service]' # the CLI + daemon; core CLI only: pip insta
|
|
|
110
110
|
# more extras: [adbc] postgres/redshift · [spark] · [polars] · [all]
|
|
111
111
|
```
|
|
112
112
|
|
|
113
|
-
> **Platforms.** Developed on Linux
|
|
114
|
-
>
|
|
115
|
-
>
|
|
116
|
-
>
|
|
117
|
-
>
|
|
118
|
-
> this paragraph.
|
|
113
|
+
> **Platforms.** Developed on Linux. CI runs the full suite on Linux, and a macOS job runs
|
|
114
|
+
> `interlace init` plus `apply` on the quickstart. Nothing in the codebase is platform-specific —
|
|
115
|
+
> no `fork`, no signal handling, no POSIX-only calls, no shelling out — and every dependency
|
|
116
|
+
> ships macOS and Windows wheels. Windows is still unverified. If you run interlace on Windows,
|
|
117
|
+
> please open an issue either way; that is the fastest route to changing this paragraph.
|
|
119
118
|
|
|
120
119
|
## Sixty seconds
|
|
121
120
|
|
|
@@ -42,12 +42,11 @@ pip install 'interlaced[service]' # the CLI + daemon; core CLI only: pip insta
|
|
|
42
42
|
# more extras: [adbc] postgres/redshift · [spark] · [polars] · [all]
|
|
43
43
|
```
|
|
44
44
|
|
|
45
|
-
> **Platforms.** Developed on Linux
|
|
46
|
-
>
|
|
47
|
-
>
|
|
48
|
-
>
|
|
49
|
-
>
|
|
50
|
-
> this paragraph.
|
|
45
|
+
> **Platforms.** Developed on Linux. CI runs the full suite on Linux, and a macOS job runs
|
|
46
|
+
> `interlace init` plus `apply` on the quickstart. Nothing in the codebase is platform-specific —
|
|
47
|
+
> no `fork`, no signal handling, no POSIX-only calls, no shelling out — and every dependency
|
|
48
|
+
> ships macOS and Windows wheels. Windows is still unverified. If you run interlace on Windows,
|
|
49
|
+
> please open an issue either way; that is the fastest route to changing this paragraph.
|
|
51
50
|
|
|
52
51
|
## Sixty seconds
|
|
53
52
|
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "interlaced"
|
|
7
|
-
version = "2.8.
|
|
7
|
+
version = "2.8.3"
|
|
8
8
|
description = "Python and SQL models in one DAG. Transformation, orchestration and durable streaming in one process, on DuckDB and Postgres."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.12"
|
|
@@ -113,11 +113,11 @@ select = [
|
|
|
113
113
|
"B", # flake8-bugbear
|
|
114
114
|
"C4", # flake8-comprehensions
|
|
115
115
|
"UP", # pyupgrade
|
|
116
|
+
"C901", # mccabe; existing hotspots are allow-listed with noqa
|
|
116
117
|
]
|
|
117
118
|
ignore = [
|
|
118
119
|
"E501", # line too long (handled by formatter)
|
|
119
120
|
"B008", # function calls in argument defaults
|
|
120
|
-
"C901", # too complex
|
|
121
121
|
]
|
|
122
122
|
|
|
123
123
|
[tool.ruff.lint.per-file-ignores]
|
|
@@ -171,6 +171,7 @@ exclude_lines = [
|
|
|
171
171
|
|
|
172
172
|
[tool.pytest.ini_options]
|
|
173
173
|
testpaths = ["tests"]
|
|
174
|
+
pythonpath = ["tests"]
|
|
174
175
|
python_files = ["test_*.py", "*_test.py"]
|
|
175
176
|
python_classes = ["Test*"]
|
|
176
177
|
python_functions = ["test_*"]
|
|
@@ -311,7 +311,7 @@ async def _table_diff(
|
|
|
311
311
|
engines.close()
|
|
312
312
|
|
|
313
313
|
|
|
314
|
-
async def _apply(
|
|
314
|
+
async def _apply( # noqa: C901
|
|
315
315
|
environment: str,
|
|
316
316
|
path: Path,
|
|
317
317
|
select: list[str],
|
|
@@ -420,7 +420,7 @@ def restate(
|
|
|
420
420
|
asyncio.run(_execute(environment, path, select, start, end, restate=True, parallelism=parallelism))
|
|
421
421
|
|
|
422
422
|
|
|
423
|
-
async def _execute(
|
|
423
|
+
async def _execute( # noqa: C901
|
|
424
424
|
environment: str,
|
|
425
425
|
path: Path,
|
|
426
426
|
select: list[str],
|
|
@@ -170,7 +170,14 @@ async def _scheduler(environment: str, path: Path, interval: float, once: bool)
|
|
|
170
170
|
"""Same loops as ``interlace serve``: reload, tick, trim, drain, flush, CDC."""
|
|
171
171
|
from types import SimpleNamespace
|
|
172
172
|
|
|
173
|
-
from interlace.scheduler.daemon import
|
|
173
|
+
from interlace.scheduler.daemon import (
|
|
174
|
+
cdc_loop,
|
|
175
|
+
flush_once,
|
|
176
|
+
flusher_loop,
|
|
177
|
+
remember_runtime,
|
|
178
|
+
scheduler_loop,
|
|
179
|
+
source_mtime,
|
|
180
|
+
)
|
|
174
181
|
from interlace.streaming.materializer import quarantine_stream, stream_consumers
|
|
175
182
|
|
|
176
183
|
project = Project.load(path)
|
|
@@ -209,6 +216,7 @@ async def _scheduler(environment: str, path: Path, interval: float, once: bool)
|
|
|
209
216
|
stream_consumer_map={name: sorted(stream_consumers(compiled, name)) for name in streams},
|
|
210
217
|
)
|
|
211
218
|
host.flushed_heads = dict(host.log_heads)
|
|
219
|
+
remember_runtime(host, project.config)
|
|
212
220
|
flusher: asyncio.Task[None] | None = None
|
|
213
221
|
cdc: asyncio.Task[None] | None = None
|
|
214
222
|
try:
|
|
@@ -243,7 +243,7 @@ def _window(value: str, flag: str) -> datetime | None:
|
|
|
243
243
|
raise typer.Exit(2) from exc
|
|
244
244
|
|
|
245
245
|
|
|
246
|
-
def _render(plan: Plan, environment: str) -> None:
|
|
246
|
+
def _render(plan: Plan, environment: str) -> None: # noqa: C901
|
|
247
247
|
if not plan.changes and not plan.physical and not plan.transfers and not plan.drift and not plan.warnings:
|
|
248
248
|
console.print(f"No changes for [bold]{environment}[/bold].")
|
|
249
249
|
return
|
|
@@ -13,6 +13,7 @@ left literal so a missing one surfaces as an obvious ``${VAR}`` in errors.
|
|
|
13
13
|
|
|
14
14
|
from __future__ import annotations
|
|
15
15
|
|
|
16
|
+
import datetime
|
|
16
17
|
import os
|
|
17
18
|
import re
|
|
18
19
|
from pathlib import Path
|
|
@@ -143,6 +144,9 @@ class InputConfig(BaseModel):
|
|
|
143
144
|
format: Literal["parquet", "csv", "json", "delta", "iceberg"]
|
|
144
145
|
path: str
|
|
145
146
|
connection: str | None = None # http connection or an engine secret, for remote paths
|
|
147
|
+
# Hash the local file bytes into every model that reads this input, so an edit
|
|
148
|
+
# is a plan change. ``schedule: {watch:}`` only re-runs; it does not do this.
|
|
149
|
+
watch: bool = False
|
|
146
150
|
|
|
147
151
|
@model_validator(mode="after")
|
|
148
152
|
def _env_resolved(self) -> InputConfig:
|
|
@@ -154,8 +158,9 @@ class EngineConfig(BaseModel):
|
|
|
154
158
|
"""One named execution engine (warehouse gateway).
|
|
155
159
|
|
|
156
160
|
DuckDB-family types (``duckdb`` / ``ducklake`` / ``quack``) and ``postgres`` are
|
|
157
|
-
the tested set. ``spark`` is beta
|
|
158
|
-
|
|
161
|
+
the tested set. ``spark`` is beta. ``snowflake`` is alpha and has been exercised
|
|
162
|
+
against one account (not in CI). ``motherduck`` / ``redshift`` / ``bigquery`` are
|
|
163
|
+
alpha (wired and dialect-correct, not yet live-validated).
|
|
159
164
|
"""
|
|
160
165
|
|
|
161
166
|
type: str = "duckdb"
|
|
@@ -181,6 +186,60 @@ class EngineConfig(BaseModel):
|
|
|
181
186
|
return _TYPE_DIALECT.get(self.type, "duckdb")
|
|
182
187
|
|
|
183
188
|
|
|
189
|
+
class VarConfig(BaseModel):
|
|
190
|
+
"""A typed value a SQL model reads with ``var('name')``.
|
|
191
|
+
|
|
192
|
+
The call is replaced in the AST before fingerprinting, so editing ``value``
|
|
193
|
+
rebuilds every model that references the name. ``${VAR}`` in the YAML is
|
|
194
|
+
already expanded by the time this is parsed.
|
|
195
|
+
"""
|
|
196
|
+
|
|
197
|
+
type: Literal["string", "int", "float", "bool", "date", "timestamp"]
|
|
198
|
+
value: str | int | float | bool
|
|
199
|
+
|
|
200
|
+
@model_validator(mode="after")
|
|
201
|
+
def _value_matches_type(self) -> VarConfig:
|
|
202
|
+
kind = self.type
|
|
203
|
+
value = self.value
|
|
204
|
+
if kind == "string":
|
|
205
|
+
if not isinstance(value, str):
|
|
206
|
+
raise ValueError("string var value must be a string")
|
|
207
|
+
elif kind == "int":
|
|
208
|
+
if type(value) is not int:
|
|
209
|
+
raise ValueError("int var value must be an integer")
|
|
210
|
+
elif kind == "float":
|
|
211
|
+
if type(value) not in (int, float):
|
|
212
|
+
raise ValueError("float var value must be a number")
|
|
213
|
+
elif kind == "bool":
|
|
214
|
+
if type(value) is not bool:
|
|
215
|
+
raise ValueError("bool var value must be true or false")
|
|
216
|
+
elif kind == "date":
|
|
217
|
+
self.value = _as_date(value)
|
|
218
|
+
else:
|
|
219
|
+
self.value = _as_timestamp(value)
|
|
220
|
+
return self
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def _as_date(value: object) -> str:
|
|
224
|
+
if isinstance(value, str):
|
|
225
|
+
try:
|
|
226
|
+
datetime.date.fromisoformat(value)
|
|
227
|
+
except ValueError as exc:
|
|
228
|
+
raise ValueError(f"date var value must be YYYY-MM-DD, got {value!r}") from exc
|
|
229
|
+
return value
|
|
230
|
+
raise ValueError("date var value must be a YYYY-MM-DD string")
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def _as_timestamp(value: object) -> str:
|
|
234
|
+
if isinstance(value, str):
|
|
235
|
+
try:
|
|
236
|
+
datetime.datetime.fromisoformat(value)
|
|
237
|
+
except ValueError as exc:
|
|
238
|
+
raise ValueError(f"timestamp var value must be ISO-8601, got {value!r}") from exc
|
|
239
|
+
return value
|
|
240
|
+
raise ValueError("timestamp var value must be an ISO-8601 string")
|
|
241
|
+
|
|
242
|
+
|
|
184
243
|
class ProjectConfig(BaseModel):
|
|
185
244
|
"""Top-level project settings."""
|
|
186
245
|
|
|
@@ -234,10 +293,20 @@ class ProjectConfig(BaseModel):
|
|
|
234
293
|
connections: dict[str, ConnectionConfig] = Field(default_factory=dict)
|
|
235
294
|
# External files DuckDB scans as relations models can FROM. Other engines reject them.
|
|
236
295
|
inputs: dict[str, InputConfig] = Field(default_factory=dict)
|
|
296
|
+
# Typed literals SQL models read with var('name'). Expanded into the AST at
|
|
297
|
+
# compile time, so a changed value is a plan change for the models that use it.
|
|
298
|
+
vars: dict[str, VarConfig] = Field(default_factory=dict)
|
|
237
299
|
# Postgres logical replication into a declared @stream. Empty means the daemon
|
|
238
300
|
# does not open a replication connection.
|
|
239
301
|
cdc: dict[str, CdcConfig] = Field(default_factory=dict)
|
|
240
302
|
|
|
303
|
+
@model_validator(mode="after")
|
|
304
|
+
def _var_names_are_identifiers(self) -> ProjectConfig:
|
|
305
|
+
for name in self.vars:
|
|
306
|
+
if not _ENV_KEY.fullmatch(name):
|
|
307
|
+
raise ValueError(f"var {name!r} must be an identifier")
|
|
308
|
+
return self
|
|
309
|
+
|
|
241
310
|
@model_validator(mode="after")
|
|
242
311
|
def _cdc_uses_postgres(self) -> ProjectConfig:
|
|
243
312
|
for name, source in self.cdc.items():
|
|
@@ -95,7 +95,7 @@ def render_model_sql(definition: ModelDef) -> str:
|
|
|
95
95
|
return f"/*\n{header}\n*/\n{definition.sql.strip()}\n"
|
|
96
96
|
|
|
97
97
|
|
|
98
|
-
def model_to_config(definition: ModelDef) -> dict[str, Any]:
|
|
98
|
+
def model_to_config(definition: ModelDef) -> dict[str, Any]: # noqa: C901
|
|
99
99
|
"""The ``interlace:`` mapping ``_sql_model`` reads. Defaults are omitted."""
|
|
100
100
|
config: dict[str, Any] = {"name": definition.name}
|
|
101
101
|
_keep(config, "materialise", definition.materialise, MATERIALISE_DEFAULT)
|
|
@@ -22,11 +22,11 @@ from typing import Any
|
|
|
22
22
|
import pyarrow as pa
|
|
23
23
|
from sqlglot import exp
|
|
24
24
|
|
|
25
|
-
from interlace.engines.base import EngineAdapter, EngineCaps, LoadMode, note_statement
|
|
25
|
+
from interlace.engines.base import EngineAdapter, EngineCaps, LoadMode, note_statement, relation_is_absent
|
|
26
26
|
from interlace.ir.relation import TableRef
|
|
27
27
|
|
|
28
28
|
|
|
29
|
-
def arrow_type_name(dtype: pa.DataType) -> str:
|
|
29
|
+
def arrow_type_name(dtype: pa.DataType) -> str: # noqa: C901
|
|
30
30
|
"""A canonical SQL type name for an Arrow type, in the vocabulary the planner's
|
|
31
31
|
alignment/widening logic understands (see ``plan.apply``)."""
|
|
32
32
|
if pa.types.is_boolean(dtype):
|
|
@@ -162,6 +162,8 @@ class AdbcAdapter(EngineAdapter):
|
|
|
162
162
|
with self._lock:
|
|
163
163
|
try:
|
|
164
164
|
schema = self._conn.adbc_get_table_schema(table.name, db_schema_filter=table.schema)
|
|
165
|
-
except Exception
|
|
166
|
-
|
|
165
|
+
except Exception as exc:
|
|
166
|
+
if relation_is_absent(exc):
|
|
167
|
+
return {}
|
|
168
|
+
raise
|
|
167
169
|
return {field.name: arrow_type_name(field.type) for field in schema}
|
|
@@ -42,6 +42,19 @@ def note_statement(exc: BaseException, sql: str) -> None:
|
|
|
42
42
|
return
|
|
43
43
|
|
|
44
44
|
|
|
45
|
+
def relation_is_absent(exc: BaseException) -> bool:
|
|
46
|
+
"""Whether ``exc`` means the relation is missing, not that the probe itself failed.
|
|
47
|
+
|
|
48
|
+
Catalog helpers treat a missing table as an empty description. Any other
|
|
49
|
+
error has to surface: swallowing it makes a broken catalog look like no columns.
|
|
50
|
+
"""
|
|
51
|
+
message = str(exc).casefold()
|
|
52
|
+
if "table function" in message:
|
|
53
|
+
return False
|
|
54
|
+
markers = ("does not exist", "doesn't exist", "not found", "no such table", "undefined table", "unknown table")
|
|
55
|
+
return any(marker in message for marker in markers)
|
|
56
|
+
|
|
57
|
+
|
|
45
58
|
def statement_of(exc: BaseException) -> str | None:
|
|
46
59
|
"""The SQL attached to ``exc`` or to the error it was raised from."""
|
|
47
60
|
seen: set[int] = set()
|
|
@@ -163,7 +176,9 @@ class EngineAdapter(ABC):
|
|
|
163
176
|
|
|
164
177
|
def transpile(self, ast: exp.Expr) -> str:
|
|
165
178
|
"""Canonical AST -> this engine's SQL. The one place dialect leaks back in."""
|
|
166
|
-
|
|
179
|
+
from interlace.ir.dialect import render_sql
|
|
180
|
+
|
|
181
|
+
return render_sql(ast, self.dialect)
|
|
167
182
|
|
|
168
183
|
def refresh_inputs(self) -> None:
|
|
169
184
|
"""Re-expand input views whose paths contain date tokens.
|
|
@@ -178,3 +193,7 @@ class EngineAdapter(ABC):
|
|
|
178
193
|
def attach(self, alias: str, uri: str) -> None:
|
|
179
194
|
"""ATTACH another database under ``alias``. Only engines with ``supports_attach``."""
|
|
180
195
|
raise NotImplementedError(f"{type(self).__name__} cannot ATTACH another database")
|
|
196
|
+
|
|
197
|
+
@abstractmethod
|
|
198
|
+
def close(self) -> None:
|
|
199
|
+
"""Release the connection this adapter holds."""
|
|
@@ -39,7 +39,7 @@ import pyarrow as pa
|
|
|
39
39
|
import tenacity
|
|
40
40
|
from sqlglot import exp
|
|
41
41
|
|
|
42
|
-
from interlace.engines.base import EngineAdapter, EngineCaps, LoadMode, note_statement
|
|
42
|
+
from interlace.engines.base import EngineAdapter, EngineCaps, LoadMode, note_statement, relation_is_absent
|
|
43
43
|
from interlace.exceptions import ConfigurationError
|
|
44
44
|
from interlace.ir.relation import TableRef
|
|
45
45
|
|
|
@@ -307,8 +307,10 @@ class DuckDBAdapter(EngineAdapter):
|
|
|
307
307
|
)
|
|
308
308
|
try:
|
|
309
309
|
reader = await self.fetch_sql(sql)
|
|
310
|
-
except Exception:
|
|
311
|
-
|
|
310
|
+
except Exception as exc:
|
|
311
|
+
if relation_is_absent(exc):
|
|
312
|
+
return []
|
|
313
|
+
raise
|
|
312
314
|
return [str(row[column]) for row in reader.read_all().to_pylist() if row.get(column)]
|
|
313
315
|
|
|
314
316
|
# --- sync workers (run in a thread) -------------------------------------
|
|
@@ -20,7 +20,7 @@ from __future__ import annotations
|
|
|
20
20
|
from sqlglot import exp
|
|
21
21
|
|
|
22
22
|
from interlace.engines.adbc import AdbcAdapter
|
|
23
|
-
from interlace.engines.base import EngineCaps
|
|
23
|
+
from interlace.engines.base import EngineCaps, relation_is_absent
|
|
24
24
|
from interlace.exceptions import ConfigurationError
|
|
25
25
|
from interlace.ir.relation import TableRef
|
|
26
26
|
|
|
@@ -83,8 +83,10 @@ class PostgresAdapter(AdbcAdapter):
|
|
|
83
83
|
reader = await self.fetch_sql(
|
|
84
84
|
f"SELECT indexname FROM pg_indexes WHERE schemaname = {schema} AND tablename = {name}"
|
|
85
85
|
)
|
|
86
|
-
except Exception:
|
|
87
|
-
|
|
86
|
+
except Exception as exc:
|
|
87
|
+
if relation_is_absent(exc):
|
|
88
|
+
return []
|
|
89
|
+
raise
|
|
88
90
|
return [str(row["indexname"]) for row in reader.read_all().to_pylist() if row.get("indexname")]
|
|
89
91
|
|
|
90
92
|
async def list_constraints(self, table: TableRef) -> list[str]:
|
|
@@ -97,6 +99,8 @@ class PostgresAdapter(AdbcAdapter):
|
|
|
97
99
|
"JOIN pg_namespace nsp ON nsp.oid = rel.relnamespace "
|
|
98
100
|
f"WHERE nsp.nspname = {schema} AND rel.relname = {name}"
|
|
99
101
|
)
|
|
100
|
-
except Exception:
|
|
101
|
-
|
|
102
|
+
except Exception as exc:
|
|
103
|
+
if relation_is_absent(exc):
|
|
104
|
+
return []
|
|
105
|
+
raise
|
|
102
106
|
return [str(row["name"]) for row in reader.read_all().to_pylist() if row.get("name")]
|
|
@@ -33,6 +33,7 @@ class EngineRegistry(Mapping[str, EngineAdapter]):
|
|
|
33
33
|
*,
|
|
34
34
|
default: str = "default",
|
|
35
35
|
attach_uris: Mapping[str, str] | None = None,
|
|
36
|
+
sinks: Mapping[str, str] | None = None,
|
|
36
37
|
) -> None:
|
|
37
38
|
self._names = frozenset(names)
|
|
38
39
|
self._opener = opener
|
|
@@ -41,6 +42,9 @@ class EngineRegistry(Mapping[str, EngineAdapter]):
|
|
|
41
42
|
# engine name -> URI another DuckDB-family engine could ATTACH (transfer
|
|
42
43
|
# fast lane); absent/None = only reachable through its own adapter.
|
|
43
44
|
self.attach_uris: dict[str, str] = dict(attach_uris or {})
|
|
45
|
+
# reverse-ETL alias -> database URI. A DuckDB warehouse ATTACHes these.
|
|
46
|
+
# An engine that cannot ATTACH opens the URI and writes the delivery there.
|
|
47
|
+
self.sinks: dict[str, str] = dict(sinks or {})
|
|
44
48
|
if default not in self._names:
|
|
45
49
|
raise ConfigurationError(
|
|
46
50
|
f"default_engine {default!r} is not a configured engine",
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
"""Snowflake engine adapter (ALPHA — exercised against one account, not in CI).
|
|
2
|
+
|
|
3
|
+
An ADBC-transport engine: it inherits all of :class:`~interlace.engines.adbc.AdbcAdapter`
|
|
4
|
+
(execute / fetch → Arrow / ``adbc_ingest`` bulk load / metadata ``describe``) and
|
|
5
|
+
only sets the dialect and capabilities. Snowflake supports the full strategy set:
|
|
6
|
+
``CREATE OR REPLACE TABLE``, ``SELECT * EXCLUDE`` (so scd works with ``SELECT *``),
|
|
7
|
+
and a native ``MERGE``.
|
|
8
|
+
|
|
9
|
+
Snowflake folds unquoted identifiers to uppercase and returns those names. Fetch
|
|
10
|
+
and describe fold them back, so contracts and Python models keep the names the
|
|
11
|
+
SQL was written with. ``adbc_ingest`` on this driver rejects a target schema, so
|
|
12
|
+
loads ``USE SCHEMA`` first.
|
|
13
|
+
|
|
14
|
+
Requires the ``adbc-snowflake`` extra. The connection string is the Snowflake ADBC
|
|
15
|
+
URI (``user[:password]@account/database/schema?warehouse=WH&role=R``); key-pair and
|
|
16
|
+
external-browser auth go through ``db_kwargs``.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import pyarrow as pa
|
|
22
|
+
from sqlglot import exp
|
|
23
|
+
|
|
24
|
+
from interlace.engines.adbc import AdbcAdapter
|
|
25
|
+
from interlace.engines.base import EngineCaps, LoadMode
|
|
26
|
+
from interlace.exceptions import ConfigurationError
|
|
27
|
+
from interlace.ir.relation import TableRef
|
|
28
|
+
|
|
29
|
+
_SNOWFLAKE_CAPS = EngineCaps(
|
|
30
|
+
supports_create_or_replace=True,
|
|
31
|
+
supports_star_exclude=True, # SELECT * EXCLUDE (...)
|
|
32
|
+
supports_merge=True,
|
|
33
|
+
supports_transactions=True,
|
|
34
|
+
# PRIMARY KEY / UNIQUE / FOREIGN KEY are informational. NOT NULL is enforced.
|
|
35
|
+
enforced_constraints=frozenset({"not_null"}),
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class SnowflakeAdapter(AdbcAdapter):
|
|
40
|
+
"""Executes canonical ASTs inside Snowflake; Arrow in and out via ADBC."""
|
|
41
|
+
|
|
42
|
+
dialect = "snowflake"
|
|
43
|
+
caps = _SNOWFLAKE_CAPS
|
|
44
|
+
|
|
45
|
+
@classmethod
|
|
46
|
+
def connect(cls, dsn: str) -> SnowflakeAdapter:
|
|
47
|
+
try:
|
|
48
|
+
import adbc_driver_snowflake.dbapi as dbapi
|
|
49
|
+
except ImportError as exc: # pragma: no cover - import guard
|
|
50
|
+
raise ConfigurationError(
|
|
51
|
+
"the snowflake engine needs the 'adbc-snowflake' extra: pip install 'interlaced[adbc-snowflake]'"
|
|
52
|
+
) from exc
|
|
53
|
+
return cls(dbapi.connect(dsn))
|
|
54
|
+
|
|
55
|
+
def _fetch_sync(self, sql: str) -> pa.RecordBatchReader:
|
|
56
|
+
table = super()._fetch_sync(sql).read_all()
|
|
57
|
+
return table.rename_columns([name.casefold() for name in table.column_names]).to_reader()
|
|
58
|
+
|
|
59
|
+
def _describe_sync(self, table: TableRef) -> dict[str, str]:
|
|
60
|
+
# The driver quotes the name it is given. Unquoted SQL folds to uppercase,
|
|
61
|
+
# so a lowercase lookup looks for a different table and comes back empty.
|
|
62
|
+
folded = TableRef(
|
|
63
|
+
schema=table.schema.upper(),
|
|
64
|
+
name=table.name.upper(),
|
|
65
|
+
catalog=table.catalog.upper() if table.catalog else None,
|
|
66
|
+
)
|
|
67
|
+
described = super()._describe_sync(folded)
|
|
68
|
+
return {name.casefold(): kind for name, kind in described.items()}
|
|
69
|
+
|
|
70
|
+
def _load_sync(self, table: TableRef, reader: pa.RecordBatchReader, mode: LoadMode) -> int:
|
|
71
|
+
# This driver rejects adbc.ingest.target_db_schema. The session schema is
|
|
72
|
+
# the ingest target; unquoted names fold the same way the model's SQL does.
|
|
73
|
+
ingest_mode = "replace" if mode == "create" else "append"
|
|
74
|
+
schema = exp.to_identifier(table.schema).sql(dialect=self.dialect) if table.schema else ""
|
|
75
|
+
# Arrow names are the SQL as written (lowercase). The driver quotes them,
|
|
76
|
+
# so they must be the uppercase form unquoted SQL folds to.
|
|
77
|
+
loaded_table = reader.read_all()
|
|
78
|
+
batch = loaded_table.rename_columns([name.upper() for name in loaded_table.column_names])
|
|
79
|
+
with self._lock:
|
|
80
|
+
try:
|
|
81
|
+
with self._conn.cursor() as cur:
|
|
82
|
+
if schema:
|
|
83
|
+
cur.execute(f"USE SCHEMA {schema}")
|
|
84
|
+
loaded = cur.adbc_ingest(table.name.upper(), batch, mode=ingest_mode)
|
|
85
|
+
self._conn.commit()
|
|
86
|
+
except Exception:
|
|
87
|
+
self._conn.rollback()
|
|
88
|
+
raise
|
|
89
|
+
return int(loaded) if isinstance(loaded, int) and loaded > 0 else 0
|
|
@@ -11,9 +11,9 @@ and build the snapshots it persists.
|
|
|
11
11
|
from __future__ import annotations
|
|
12
12
|
|
|
13
13
|
import inspect
|
|
14
|
-
import textwrap
|
|
15
14
|
from collections.abc import Iterable, Mapping
|
|
16
15
|
from dataclasses import dataclass, field
|
|
16
|
+
from typing import Any
|
|
17
17
|
|
|
18
18
|
from sqlglot import exp
|
|
19
19
|
|
|
@@ -22,9 +22,16 @@ from interlace.dsl.decorators import CheckDef, ModelDef, ModelFn
|
|
|
22
22
|
from interlace.exceptions import CompilationError, DefinitionError
|
|
23
23
|
from interlace.graph.dag import DependencyGraph
|
|
24
24
|
from interlace.ir.canonicalize import as_query, parse, table_references
|
|
25
|
-
from interlace.ir.fingerprint import
|
|
25
|
+
from interlace.ir.fingerprint import (
|
|
26
|
+
canonical_sql,
|
|
27
|
+
data_fingerprint,
|
|
28
|
+
metadata_fingerprint,
|
|
29
|
+
physical_fingerprint,
|
|
30
|
+
python_source,
|
|
31
|
+
)
|
|
26
32
|
from interlace.ir.macros import Macro, expand_macros
|
|
27
33
|
from interlace.ir.relation import TableRef
|
|
34
|
+
from interlace.ir.vars import TypedVar, expand_vars
|
|
28
35
|
from interlace.physical.spec import ConstraintSpec, IndexSpec, SchemaPolicy, physical_payload
|
|
29
36
|
|
|
30
37
|
# Snapshot tables live in `interlace__<logical schema>` — exclusively owned, never a
|
|
@@ -102,7 +109,11 @@ def _physical_table(name: str, fingerprint: str, catalog: str | None) -> TableRe
|
|
|
102
109
|
|
|
103
110
|
|
|
104
111
|
def _resolve_dependencies(
|
|
105
|
-
model: ModelDef,
|
|
112
|
+
model: ModelDef,
|
|
113
|
+
names: set[str],
|
|
114
|
+
default_dialect: str,
|
|
115
|
+
macros: Mapping[str, Macro],
|
|
116
|
+
variables: Mapping[str, TypedVar],
|
|
106
117
|
) -> tuple[tuple[str, ...], exp.Query | None, str]:
|
|
107
118
|
dialect = model.dialect or default_dialect
|
|
108
119
|
deps: list[str] = []
|
|
@@ -128,6 +139,9 @@ def _resolve_dependencies(
|
|
|
128
139
|
# here is what makes a macro edit rebuild its callers, and what lets a macro body
|
|
129
140
|
# reference a model and have that count as a dependency.
|
|
130
141
|
ast = as_query(expand_macros(ast, macros, model.name), what=f"model {model.name}")
|
|
142
|
+
# After macros, so a macro body may call var() and the literal lands in
|
|
143
|
+
# the fingerprint. An unknown name fails here, before the warehouse sees it.
|
|
144
|
+
ast = as_query(expand_vars(ast, variables, model.name), what=f"model {model.name}")
|
|
131
145
|
for ref in table_references(ast):
|
|
132
146
|
if ref in names:
|
|
133
147
|
add(ref)
|
|
@@ -149,11 +163,11 @@ def _fingerprint_query(model: ModelDef, ast: exp.Query | None) -> str | exp.Quer
|
|
|
149
163
|
if ast is not None:
|
|
150
164
|
return ast
|
|
151
165
|
if model.fn is not None:
|
|
152
|
-
return
|
|
166
|
+
return python_source(model.fn)
|
|
153
167
|
raise DefinitionError(f"model {model.name!r} has neither SQL nor a function body")
|
|
154
168
|
|
|
155
169
|
|
|
156
|
-
def compile_models(
|
|
170
|
+
def compile_models( # noqa: C901
|
|
157
171
|
models: Iterable[ModelDef],
|
|
158
172
|
*,
|
|
159
173
|
default_dialect: str = "duckdb",
|
|
@@ -163,6 +177,8 @@ def compile_models(
|
|
|
163
177
|
catalog: str | None = None,
|
|
164
178
|
checks: Iterable[CheckDef] = (),
|
|
165
179
|
macros: Mapping[str, Macro] | None = None,
|
|
180
|
+
input_hashes: Mapping[str, str] | None = None,
|
|
181
|
+
variables: Mapping[str, TypedVar] | None = None,
|
|
166
182
|
) -> CompiledProject:
|
|
167
183
|
"""Compile models into a fingerprinted, topologically-ordered project.
|
|
168
184
|
|
|
@@ -170,6 +186,9 @@ def compile_models(
|
|
|
170
186
|
``engine_dialects`` maps engine name → sqlglot dialect (used when a model
|
|
171
187
|
omits ``dialect``). ``known_engines`` validates model ``engine`` pins.
|
|
172
188
|
``macros`` are expanded into every model's AST before it is fingerprinted.
|
|
189
|
+
``input_hashes`` are content digests of watched inputs; a model that reads
|
|
190
|
+
one folds those digests into its fingerprint.
|
|
191
|
+
``variables`` replace ``var('name')`` with a typed literal before fingerprinting.
|
|
173
192
|
"""
|
|
174
193
|
definitions = {m.name: m for m in models}
|
|
175
194
|
names = set(definitions)
|
|
@@ -193,7 +212,9 @@ def compile_models(
|
|
|
193
212
|
)
|
|
194
213
|
# Authoring dialect: explicit model dialect, else the engine's, else project default.
|
|
195
214
|
model_default_dialect = dialects_by_engine.get(engine, default_dialect)
|
|
196
|
-
deps, ast, dialect = _resolve_dependencies(
|
|
215
|
+
deps, ast, dialect = _resolve_dependencies(
|
|
216
|
+
definition, names, model_default_dialect, macros or {}, variables or {}
|
|
217
|
+
)
|
|
197
218
|
resolved[name] = (deps, ast, dialect, engine)
|
|
198
219
|
graph.add_node(name)
|
|
199
220
|
for dep in deps:
|
|
@@ -223,7 +244,7 @@ def compile_models(
|
|
|
223
244
|
f"{compiled[dep].engine!r}; an ephemeral model must share its consumers' engine "
|
|
224
245
|
f"(see docs/architecture/MULTI_ENGINE.md)"
|
|
225
246
|
)
|
|
226
|
-
strategy_config = {
|
|
247
|
+
strategy_config: dict[str, Any] = {
|
|
227
248
|
"materialise": definition.materialise,
|
|
228
249
|
"strategy": definition.strategy,
|
|
229
250
|
"key": list(definition.key),
|
|
@@ -240,6 +261,12 @@ def compile_models(
|
|
|
240
261
|
"environments": sorted(definition.environments),
|
|
241
262
|
"dialect": dialect,
|
|
242
263
|
}
|
|
264
|
+
if input_hashes and ast is not None:
|
|
265
|
+
from interlace.inputs import referenced_inputs
|
|
266
|
+
|
|
267
|
+
watched = {name: input_hashes[name] for name in referenced_inputs(ast, set(input_hashes))}
|
|
268
|
+
if watched:
|
|
269
|
+
strategy_config["inputs"] = watched
|
|
243
270
|
query = _fingerprint_query(definition, ast)
|
|
244
271
|
# render the canonical SQL once: it feeds both fingerprints and definition_sql
|
|
245
272
|
canonical = canonical_sql(query) if isinstance(query, exp.Expr) else query
|