wherobots-dbt 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. dbt/adapters/wherobots/__init__.py +35 -0
  2. dbt/adapters/wherobots/__version__.py +11 -0
  3. dbt/include/wherobots/__init__.py +5 -0
  4. dbt/include/wherobots/dbt_project.yml +3 -0
  5. dbt/include/wherobots/macros/.gitkeep +0 -0
  6. dbt/include/wherobots/macros/adapters.sql +240 -0
  7. dbt/include/wherobots/macros/create_table_as.sql +97 -0
  8. dbt/include/wherobots/macros/create_view_as.sql +21 -0
  9. dbt/include/wherobots/macros/materializations/incremental/incremental.sql +173 -0
  10. dbt/include/wherobots/macros/materializations/incremental/on_schema_change.sql +38 -0
  11. dbt/include/wherobots/macros/materializations/incremental/strategies/append.sql +86 -0
  12. dbt/include/wherobots/macros/materializations/incremental/strategies/delete_insert.sql +57 -0
  13. dbt/include/wherobots/macros/materializations/incremental/strategies/insert_overwrite.sql +89 -0
  14. dbt/include/wherobots/macros/materializations/incremental/strategies/merge.sql +78 -0
  15. dbt/include/wherobots/macros/materializations/seeds/helpers.sql +90 -0
  16. dbt/include/wherobots/macros/materializations/snapshot/check_time_data_types.sql +60 -0
  17. dbt/include/wherobots/macros/materializations/snapshot/current_timestamp.sql +16 -0
  18. dbt/include/wherobots/macros/materializations/snapshot/get_true_sql.sql +11 -0
  19. dbt/include/wherobots/macros/materializations/snapshot/hash_arguments.sql +18 -0
  20. dbt/include/wherobots/macros/materializations/snapshot/post_snapshot.sql +11 -0
  21. dbt/include/wherobots/macros/materializations/snapshot/snapshot_merge.sql +47 -0
  22. dbt/include/wherobots/macros/materializations/table.sql +50 -0
  23. dbt/include/wherobots/macros/materializations/view.sql +54 -0
  24. dbt/include/wherobots/profile_template.yml +27 -0
  25. dbt_wherobots/__init__.py +14 -0
  26. dbt_wherobots/_framework_gaps.py +88 -0
  27. dbt_wherobots/_sql_literals.py +26 -0
  28. dbt_wherobots/capabilities.py +21 -0
  29. dbt_wherobots/client/__init__.py +5 -0
  30. dbt_wherobots/client/fake.py +131 -0
  31. dbt_wherobots/client/http.py +573 -0
  32. dbt_wherobots/column.py +40 -0
  33. dbt_wherobots/connections.py +400 -0
  34. dbt_wherobots/credentials.py +147 -0
  35. dbt_wherobots/impl.py +746 -0
  36. dbt_wherobots/ports.py +141 -0
  37. dbt_wherobots/relation.py +39 -0
  38. dbt_wherobots/types/__init__.py +343 -0
  39. wherobots_dbt-0.1.0.dist-info/METADATA +72 -0
  40. wherobots_dbt-0.1.0.dist-info/RECORD +42 -0
  41. wherobots_dbt-0.1.0.dist-info/WHEEL +4 -0
  42. wherobots_dbt-0.1.0.dist-info/licenses/LICENSE +201 -0
@@ -0,0 +1,35 @@
1
+ """dbt plugin shim for the Wherobots adapter.
2
+
3
+ dbt discovers adapters at ``dbt.adapters.<name>``, but the implementation lives in the top-level
4
+ ``dbt_wherobots`` package (see that package's docstring for why). This module is the thin bridge: it
5
+ exposes the dbt ``AdapterPlugin`` wiring the adapter, the credentials, and the include package path.
6
+
7
+ No ``dependencies`` are declared (clean-room): the adapter dispatches to ``default__`` macros in
8
+ dbt's global project rather than depending on another adapter's project. The plugin's
9
+ ``project_name`` defaults to ``dbt_<include-dir-name>`` (``dbt_wherobots``), which must equal the
10
+ ``name`` in ``dbt/include/wherobots/dbt_project.yml`` so dbt loads the macro library (plan §10 F1).
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ from typing import TYPE_CHECKING, Any, cast
16
+
17
+ from dbt.adapters.base import AdapterPlugin
18
+
19
+ from dbt.include import wherobots
20
+ from dbt_wherobots.credentials import WherobotsCredentials
21
+ from dbt_wherobots.impl import WherobotsAdapter
22
+
23
+ if TYPE_CHECKING:
24
+ from dbt.adapters.protocol import AdapterProtocol
25
+
26
+ # dbt's ``AdapterProtocol`` (TODO CT-211 upstream) declares connection-manager methods such as
27
+ # ``set_query_header`` directly on the adapter, so no real ``SQLAdapter`` subclass satisfies it
28
+ # structurally — every adapter only delegates to its manager. ``cast`` is the typed bridge across
29
+ # that upstream imprecision (not an inline suppression): ``WherobotsAdapter`` is the concrete
30
+ # adapter the registry instantiates.
31
+ Plugin = AdapterPlugin(
32
+ adapter=cast("type[AdapterProtocol[Any, Any, Any, Any]]", WherobotsAdapter),
33
+ credentials=WherobotsCredentials,
34
+ include_path=str(wherobots.PACKAGE_PATH),
35
+ )
@@ -0,0 +1,11 @@
1
+ """Adapter version for dbt's plugin registry.
2
+
3
+ dbt's adapter factory reads ``dbt.adapters.<name>.__version__.version`` (falling back from
4
+ ``__about__``) to record the adapter version in run metadata. dbt validates this as a strict
5
+ semantic version (``dbt_common.semver``), so it tracks the release line of the package version in
6
+ ``pyproject.toml`` without the PEP 440 ``.devN`` suffix that semver rejects.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ version = "0.1.0"
@@ -0,0 +1,5 @@
1
+ """dbt include package for wherobots (macros + project scaffolding land in M1+)."""
2
+
3
+ from pathlib import Path
4
+
5
+ PACKAGE_PATH = Path(__file__).parent
@@ -0,0 +1,3 @@
1
+ name: dbt_wherobots
2
+ config-version: 2
3
+ macro-paths: ["macros"]
File without changes
@@ -0,0 +1,240 @@
1
+ {#
2
+ Wherobots dialect macros (component #7) — schema/relation/column metadata.
3
+
4
+ Pinned to the live-wire facts (plan §3) and ADR-0007a (stateless, backtick-quoted,
5
+ fully-qualified rendering — no USE). SHOW/DESCRIBE parsing lives here in Jinja by design
6
+ (plan §4); cosmic-ray mutates Python only, so the acceptance/functional assertions are the
7
+ sole guard on these macros (plan §10 F6) and are deliberately discriminating.
8
+ #}
9
+
10
+
11
+ {% macro wherobots__create_schema(relation) -%}
12
+ {#- Fact 1: a fully-qualified create succeeds cold; `if not exists` is idempotent. -#}
13
+ {%- call statement('create_schema') -%}
14
+ create schema if not exists {{ relation.without_identifier() }}
15
+ {%- endcall -%}
16
+ {%- endmacro %}
17
+
18
+
19
+ {% macro wherobots__drop_schema(relation) -%}
20
+ {#- Fact 10: dropping an empty schema needs no CASCADE (dbt drops relations first). -#}
21
+ {%- call statement('drop_schema') -%}
22
+ drop schema if exists {{ relation.without_identifier() }}
23
+ {%- endcall -%}
24
+ {%- endmacro %}
25
+
26
+
27
+ {% macro wherobots__drop_relation(relation) -%}
28
+ {#-
29
+ V4 / ADR-0010(b): DROP is type-segregated — a wrong-type `drop ... if exists` is a SILENT NO-OP
30
+ (the object survives — a leak). So branch on `relation.is_view` to emit `drop view if exists`
31
+ for a view and `drop table if exists` for a table. Both are idempotent with `if exists` (V6 — a
32
+ missing object is acknowledged). The relation type comes from the typed cache the `show tables`
33
+ ∪ `show views` union (`list_relations_without_caching`) now populates; a null-type relation
34
+ never reaches this macro because the base `SQLAdapter.drop_relation` null-type guard is restored
35
+ (the M1 override is removed — §4.4a / ADR-0010(b)).
36
+ -#}
37
+ {%- set relation_type = 'view' if relation.is_view else 'table' -%}
38
+ {%- call statement('drop_relation') -%}
39
+ drop {{ relation_type }} if exists {{ relation }}
40
+ {%- endcall -%}
41
+ {%- endmacro %}
42
+
43
+
44
+ {% macro wherobots__rename_relation(from_relation, to_relation) -%}
45
+ {#-
46
+ Fact 4: ALTER TABLE ... RENAME TO works ONLY with a BARE single-identifier target — a
47
+ fully-qualified target silently strips the catalog and fails TABLE_OR_VIEW_NOT_FOUND. So the
48
+ target is rendered as the bare identifier, never the three-part name.
49
+ -#}
50
+ {%- call statement('rename_relation') -%}
51
+ alter table {{ from_relation }} rename to {{ to_relation.identifier }}
52
+ {%- endcall -%}
53
+ {%- endmacro %}
54
+
55
+
56
+ {% macro wherobots__list_schemas(database) -%}
57
+ {#- Fact 6: `show schemas in <catalog>` returns the namespace at index 0; the base
58
+ `list_schemas` reads row[0], so this macro just returns the result table.
59
+ `database` is interpolated AS-IS, never re-quoted: on the real `dbt run` path dbt passes it
60
+ already backtick-quoted, so `adapter.quote(database)` would DOUBLE-quote it (`` ``cat`` ``)
61
+ and the engine rejects that with PARSE_SYNTAX_ERROR (caught live by the conformance run).
62
+ The already-quoted (live) and unquoted (direct test-call) forms both parse (F2). -#}
63
+ {%- call statement('list_schemas', fetch_result=True) -%}
64
+ show schemas in {{ database }}
65
+ {%- endcall -%}
66
+ {{ return(load_result('list_schemas').table) }}
67
+ {%- endmacro %}
68
+
69
+
70
+ {% macro wherobots__list_relations_without_caching(schema_relation) -%}
71
+ {#-
72
+ V2 / ADR-0010(b): catalog visibility is DISJOINT by type — `show tables` lists tables ONLY and
73
+ `show views` lists views ONLY (each ['namespace','name','isTemporary'], identifier at INDEX 1,
74
+ namespace at index 0). So the relation list is the UNION of both lenses, tagging each row with
75
+ its type (`table` / `view`) — NOT a subtraction, no double-counting. The type must round-trip
76
+ into the relation cache so dbt drops/replaces each object with the right DDL (type-aware
77
+ drop_relation, V4) and a table↔view switch can pre-drop the stale object (V7).
78
+
79
+ TEMPORARY rows are SKIPPED (isTemporary at index 2, a real boolean through the port's agate
80
+ boundary): Spark lists session temp views in BOTH lenses with an EMPTY namespace, and our own
81
+ incremental materialization creates such views (`<model>__dbt_tmp_<hash>`, ADR-0012(c),
82
+ ADR-0028). They are
83
+ session-scoped, not namespace objects — caching them corrupts every cache consumer
84
+ (get_relation, type-aware drops), and `drop_schema`'s emptying loop would render them with an
85
+ empty schema segment (`` `cat`.``.`name` ``), which the live Iceberg REST catalog rejects with
86
+ ForbiddenException (caught live: CI run 27393162479 teardowns).
87
+
88
+ Returns rows shaped (database, identifier, schema, type) so the base SQLAdapter
89
+ `list_relations_without_caching` can build the relations (it consumes the 4-tuple type as-is —
90
+ no Python row-shape change needed, plan §4.5 F3a).
91
+ -#}
92
+ {%- set relations = [] -%}
93
+ {%- call statement('list_tables', fetch_result=True) -%}
94
+ show tables in {{ schema_relation }}
95
+ {%- endcall -%}
96
+ {%- for row in load_result('list_tables').table -%}
97
+ {%- if not row[2] -%}
98
+ {%- do relations.append((schema_relation.database, row[1], row[0], 'table')) -%}
99
+ {%- endif -%}
100
+ {%- endfor -%}
101
+ {%- call statement('list_views', fetch_result=True) -%}
102
+ show views in {{ schema_relation }}
103
+ {%- endcall -%}
104
+ {%- for row in load_result('list_views').table -%}
105
+ {%- if not row[2] -%}
106
+ {%- do relations.append((schema_relation.database, row[1], row[0], 'view')) -%}
107
+ {%- endif -%}
108
+ {%- endfor -%}
109
+ {{ return(relations) }}
110
+ {%- endmacro %}
111
+
112
+
113
+ {% macro wherobots__make_temp_relation(base_relation, suffix) -%}
114
+ {#-
115
+ ADR-0012(c) / I1: the incremental staging relation is a session-scoped TEMPORARY VIEW, and
116
+ temp views are NOT namespaced — so the temp relation carries NO database and NO schema (a
117
+ fresh relation, not an `incorporate` of the namespaced base). Typed `view` so the type-aware
118
+ drop (V4) and any cache interaction see an honest type.
119
+
120
+ ADR-0028: the SQL session is shared by every connection on the runtime, including other
121
+ `dbt` processes, so a bare `<identifier><suffix>` collided between runs that build a
122
+ same-named model into different schemas or catalogs, and each run inserted the other's staged
123
+ rows (DB-699). The name therefore ends with the first 8 hex digits of the md5 of the base
124
+ relation's `<database>.<schema>`: `<identifier><suffix>_<hash>`, rendered bare and
125
+ backtick-quoted. Deterministic per target, so reruns and snapshots stay stable.
126
+ -#}
127
+ {%- set namespace_hash = local_md5(base_relation.database ~ '.' ~ base_relation.schema)[:8] -%}
128
+ {%- set temp_identifier = base_relation.identifier ~ suffix ~ '_' ~ namespace_hash -%}
129
+ {%- do return(api.Relation.create(identifier=temp_identifier, type='view')) -%}
130
+ {%- endmacro %}
131
+
132
+
133
+ {% macro wherobots__alter_relation_add_remove_columns(relation, add_columns, remove_columns) -%}
134
+ {#-
135
+ ADR-0012(d) / I6: the `on_schema_change='append_new_columns'` path adds columns in ONE batch
136
+ `alter table <fq> add columns (col type, ...)` (the wire-proven Spark/Iceberg shape) — not the
137
+ stock per-column `add column` loop. Column removal never reaches this macro on the supported
138
+ paths (`sync_all_columns`, the only remover, is rejected up front — §13.6); it is declared
139
+ unsupported here too so a future caller fails loudly instead of silently leaking columns.
140
+ -#}
141
+ {%- if remove_columns -%}
142
+ {{ exceptions.raise_compiler_error("Removing columns is unsupported on the Wherobots adapter: only on_schema_change='append_new_columns' (column ADDITION) is supported (ADR-0012(d)).") }}
143
+ {%- endif -%}
144
+ {%- if add_columns -%}
145
+ {%- set additions = [] -%}
146
+ {%- for column in add_columns -%}
147
+ {%- do additions.append(adapter.quote(column.name) ~ ' ' ~ column.data_type) -%}
148
+ {%- endfor -%}
149
+ {%- call statement('alter_relation_add_columns') -%}
150
+ alter table {{ relation }} add columns ({{ additions | join(', ') }})
151
+ {%- endcall -%}
152
+ {%- endif -%}
153
+ {%- endmacro %}
154
+
155
+
156
+ {% macro wherobots__check_for_schema_changes(source_relation, target_relation) -%}
157
+ {#-
158
+ ADR-0026: the stock diff, then the CRS for any geometry column the model adds. DESCRIBE reports
159
+ geometry as bare `geometry`, which the engine rejects in `add columns`. On an Iceberg V3 target
160
+ the adapter resolves it to `geometry(<srid>)`: the column's declared `data_type` in the model's
161
+ YAML, else the SRID of its values (no values and no declaration fails); on a legacy target it
162
+ fails before any statement, pointing at `--full-refresh` (it never toggles session flags: the
163
+ session is shared across threads and processes). Only `append_new_columns` adds columns: under
164
+ `fail`, dbt raises its own schema-mismatch error right after this macro, so nothing is checked
165
+ or probed (`ignore` never calls it; `sync_all_columns` is rejected up front).
166
+ -#}
167
+ {%- set changes = default__check_for_schema_changes(source_relation, target_relation) -%}
168
+ {%- set added = changes['source_not_in_target'] -%}
169
+ {%- set appending = config.get('on_schema_change') == 'append_new_columns' -%}
170
+ {%- if appending and added and adapter.has_crs_less_geometry_columns(added) -%}
171
+ {%- do adapter.require_native_geometry_table(target_relation, added) -%}
172
+ {%- set resolved = [] -%}
173
+ {%- for column in added -%}
174
+ {%- if adapter.is_crs_less_geometry(column) -%}
175
+ {%- set quoted = adapter.quote(column.name) -%}
176
+ {%- set srids = run_query('select distinct ST_SRID(' ~ quoted ~ ') as srid from ' ~ source_relation ~ ' where ' ~ quoted ~ ' is not null') -%}
177
+ {#- A `data_type: geometry(<srid>)` in the model's YAML declares the CRS (matched by name,
178
+ case-insensitively); the values are still probed so a contradiction fails here. -#}
179
+ {%- set declared = namespace(data_type=none) -%}
180
+ {%- for name, info in (model.get('columns') or {}).items() -%}
181
+ {%- if name | lower == column.name | lower -%}
182
+ {%- set declared.data_type = info.get('data_type') -%}
183
+ {%- endif -%}
184
+ {%- endfor -%}
185
+ {%- do resolved.append(adapter.added_geometry_column(column, srids.columns[0].values(), declared.data_type)) -%}
186
+ {%- else -%}
187
+ {%- do resolved.append(column) -%}
188
+ {%- endif -%}
189
+ {%- endfor -%}
190
+ {%- do changes.update({'source_not_in_target': resolved}) -%}
191
+ {%- endif -%}
192
+ {{ return(changes) }}
193
+ {%- endmacro %}
194
+
195
+
196
+ {% macro wherobots__get_catalog_relations(information_schema, relations) -%}
197
+ {#-
198
+ ADR-0018: the TARGETED `docs generate` catalog path (capability SchemaMetadataByRelations
199
+ Supported). A THIN delegation to the typed `@available` assembly on the adapter — no agate
200
+ construction in Jinja (unlintable/unmutatable) and no new SQL verb: the assembly reuses the
201
+ `show tables` ∪ `show views` union (for the per-relation type) and `describe table` (for the
202
+ columns). The base still applies `_catalog_filter_table` + the thread-per-catalog wrapper.
203
+ -#}
204
+ {{ return(adapter.assemble_catalog_for_relations(information_schema, relations)) }}
205
+ {%- endmacro %}
206
+
207
+
208
+ {% macro wherobots__get_catalog(information_schema, schemas) -%}
209
+ {#-
210
+ ADR-0018: the UNFILTERED `docs generate` fallback path (> 100 relations, or the capability
211
+ unsupported). Same thin delegation as `wherobots__get_catalog_relations`, but assembling EVERY
212
+ relation in each involved schema (no relation filter).
213
+ -#}
214
+ {{ return(adapter.assemble_catalog_for_schemas(information_schema, schemas)) }}
215
+ {%- endmacro %}
216
+
217
+
218
+ {% macro wherobots__get_columns_in_relation(relation) -%}
219
+ {#-
220
+ Fact 8: plain `describe table` returns ['col_name','data_type','comment'] — name at index 0,
221
+ type at index 1. Use plain DESCRIBE (not EXTENDED, which dumps Iceberg metadata columns and a
222
+ `# Detailed Table Information` block). Stop at the first blank or `#` row so trailing metadata
223
+ is dropped. Fact 9 / §10 F7: a null comment cell normalizes to NaN — harmless here, the build
224
+ reads only columns 0 and 1.
225
+ -#}
226
+ {%- call statement('get_columns_in_relation', fetch_result=True) -%}
227
+ describe table {{ relation }}
228
+ {%- endcall -%}
229
+ {%- set result = load_result('get_columns_in_relation').table -%}
230
+ {%- set columns = [] -%}
231
+ {%- for row in result -%}
232
+ {%- set col_name = row[0] -%}
233
+ {%- set trimmed = (col_name | string | trim) if col_name is not none else '' -%}
234
+ {%- if trimmed == '' or trimmed.startswith('#') -%}
235
+ {%- break -%}
236
+ {%- endif -%}
237
+ {%- do columns.append(api.Column(col_name, row[1])) -%}
238
+ {%- endfor -%}
239
+ {{ return(columns) }}
240
+ {%- endmacro %}
@@ -0,0 +1,97 @@
1
+ {#
2
+ wherobots__create_table_as (component #7) — the single source of the CTAS SQL.
3
+
4
+ ADR-0007c: one atomic `create or replace table <fq> as ( ... )`. No `USING` clause — a bare CTAS
5
+ produces an Iceberg table on Wherobots (fact 3), and create-or-replace truly replaces both schema
6
+ and data in one statement (fact 2). The `table` materialization reaches this only through
7
+ `get_create_table_as_sql` -> `create_table_as` dispatch, so the replace SQL lives in exactly one
8
+ place (ADR-0007c / plan §10 F3).
9
+
10
+ ADR-0012(c): `temporary=true` (the incremental staging path) emits a session-scoped
11
+ `create or replace temporary view <identifier> as ( ... )` instead — temp views are not
12
+ namespaced (I1), so the relation renders UNQUALIFIED (defensively forced via `include`, though
13
+ `wherobots__make_temp_relation` already strips database/schema). CREATE-OR-REPLACE keeps the
14
+ staging idempotent against a same-session name collision or a failure-path leftover (§13.8).
15
+
16
+ ADR-0014(a): the NON-temporary branch threads the `partition_by` model config (Template Method —
17
+ this existing hook) through `partition_by_clause` below, rendering
18
+ `create or replace table <fq> partitioned by (...) as ( ... )` — the live-proven P1/P2 shape.
19
+ Because the `table` materialization reaches this same macro, `partition_by` works on plain table
20
+ models too (accepted scope — same code path, live-covered). The temporary branch (the staging
21
+ view) never partitions — views cannot be partitioned.
22
+
23
+ ADR-0029: the table SQL lives in `create_or_replace_table_as` below, which takes the
24
+ partition spec as an argument, so the `delete+insert` staging table reuses it UNPARTITIONED (a
25
+ model that drops one of its partition columns stays valid there, ADR-0013).
26
+ #}
27
+ {% macro wherobots__create_table_as(temporary, relation, sql) -%}
28
+ {%- set sql_header = config.get('sql_header', none) -%}
29
+ {{ sql_header if sql_header is not none }}
30
+ {%- if temporary %}
31
+ create or replace temporary view {{ relation.include(database=false, schema=false) }} as (
32
+ {{ sql }}
33
+ )
34
+ {%- else %}
35
+ {{- create_or_replace_table_as(relation, sql, config.get('partition_by')) }}
36
+ {%- endif %}
37
+ {%- endmacro %}
38
+
39
+
40
+ {#
41
+ create_or_replace_table_as — UNPREFIXED helper (not a dispatch override): the one rendering of
42
+ `create or replace table <fq> [partitioned by (...)] as ( ... )`. `partition_by` is passed in, not
43
+ read from config, so a caller can build an unpartitioned table for a partitioned model (the
44
+ `delete+insert` staging table, ADR-0029). The caller emits any `sql_header`.
45
+ #}
46
+ {% macro create_or_replace_table_as(relation, sql, partition_by) -%}
47
+ {%- set partition_clause = partition_by_clause(partition_by) %}
48
+ create or replace table {{ relation }}{% if partition_clause %} {{ partition_clause }}{% endif %} as (
49
+ {{ sql }}
50
+ )
51
+ {%- endmacro %}
52
+
53
+
54
+ {#
55
+ partition_by_clause — UNPREFIXED on purpose (a helper beside its only caller, not a dispatch
56
+ override; the `wherobots__` prefix is reserved for dispatch — plan 131 §10.13).
57
+
58
+ ADR-0014(a): renders the `partition_by` model config as `partitioned by (a, b)`, or '' when the
59
+ config is unset/empty. Accepts a string or a list of strings (dbt-spark parity); values are
60
+ PASS-THROUGH — rendered unquoted and unparsed. Plain column names are wire-proven (P1); Iceberg
61
+ transforms (`days(ts)`, `bucket(16, id)`) and reserved-word/special-char column names pass
62
+ through syntactically but are documented untested-but-unblocked (§10.11). Any other truthy shape
63
+ (e.g. the dbt-bigquery dict habit) is rejected with a compiler error — it would render garbage
64
+ through the join. (The insert_overwrite strategy additionally hard-errors on an UNSET
65
+ partition_by before reaching this macro — ADR-0014(c), validated in the materialization.)
66
+ #}
67
+ {% macro partition_by_clause(partition_by) %}
68
+ {#- Review pad 138 findings 2+3: a whitespace-only STRING is unset (aligned with the
69
+ materialization's §4.4 trim); list ENTRIES must be non-empty strings — explicit garbage in
70
+ a list is a compiler error, not `partitioned by ()` engine garbage. -#}
71
+ {%- if partition_by is string -%}
72
+ {%- if (partition_by | trim) == '' -%}
73
+ {% do return('') %}
74
+ {%- endif -%}
75
+ {%- set partition_cols = [partition_by | trim] -%}
76
+ {%- elif partition_by is not mapping and partition_by is sequence -%}
77
+ {%- set partition_cols = [] -%}
78
+ {%- for entry in partition_by -%}
79
+ {%- if entry is not string or (entry | trim) == '' -%}
80
+ {%- do exceptions.raise_compiler_error(
81
+ "partition_by entries must be non-empty strings, got: " ~ (entry | string)
82
+ ) -%}
83
+ {%- endif -%}
84
+ {%- do partition_cols.append(entry | trim) -%}
85
+ {%- endfor -%}
86
+ {%- if not partition_cols -%}
87
+ {% do return('') %}
88
+ {%- endif -%}
89
+ {%- elif not partition_by -%}
90
+ {% do return('') %}
91
+ {%- else -%}
92
+ {%- do exceptions.raise_compiler_error(
93
+ "partition_by must be a string or a list of strings, got: " ~ partition_by
94
+ ) -%}
95
+ {%- endif -%}
96
+ {% do return('partitioned by (' ~ (partition_cols | join(', ')) ~ ')') %}
97
+ {% endmacro %}
@@ -0,0 +1,21 @@
1
+ {#
2
+ wherobots__create_view_as (component #7) — the single source of the view DDL.
3
+
4
+ ADR-0010(a) / V1: one atomic `create or replace view <fq> as ( ... )`, the view sibling of the
5
+ table CTAS (ADR-0007c). create-or-replace truly replaces a view in one statement (V1), so there is
6
+ no drop / intermediate / rename dance (the fully-qualified-rename foot-gun ADR-0007 rejects, fact
7
+ 4). The `view` materialization reaches this only through `get_create_view_as_sql` ->
8
+ `create_view_as` dispatch, so the replace SQL lives in exactly one place.
9
+
10
+ Unlike `create_table_as`, the dbt-core `create_view_as` signature takes no `temporary` arg
11
+ (verified against the public `global_project` view macros, plan §4.1). The leading
12
+ `{{ sql_header if ... }}` line is load-bearing whitespace — the frozen compiled-view CTAS snapshot
13
+ (ADR-0008) was hand-derived to match this byte-for-byte; do not reformat.
14
+ #}
15
+ {% macro wherobots__create_view_as(relation, sql) -%}
16
+ {%- set sql_header = config.get('sql_header', none) -%}
17
+ {{ sql_header if sql_header is not none }}
18
+ create or replace view {{ relation }} as (
19
+ {{ sql }}
20
+ )
21
+ {%- endmacro %}
@@ -0,0 +1,173 @@
1
+ {#
2
+ The `incremental` materialization for Wherobots (component #7) — ADR-0012(a).
3
+
4
+ A hand-rolled block (bent Template Method, the table/view precedent — ADR-0007c/0010a): the stock
5
+ incremental materialization swaps full-refresh builds via `make_intermediate_/backup_relation` +
6
+ `adapter.rename_relation`, and the fully-qualified rename is the exact foot-gun ADR-0007
7
+ rejected. The rename is baked into the stock block (not a dispatchable hook), so the whole block
8
+ is copied and the swap replaced:
9
+
10
+ * First build (`existing_relation is none`) AND full-refresh -> ONE atomic
11
+ `create or replace table <fq> as ( ... )` DIRECT to target (ADR-0007 / I7) — no intermediate,
12
+ no backup, no rename.
13
+ * Existing relation is a VIEW -> full-refresh ACROSS types: create-or-replace is type-scoped
14
+ (ADR-0010 V7 / I7), so the squatting view is pre-dropped (`drop view if exists <fq>` via the
15
+ type-aware drop) THEN the table is created (plan §13.1).
16
+ * Incremental update -> stage new rows into a session-scoped TEMPORARY VIEW (ADR-0012(c) / I1),
17
+ run `process_schema_changes` for `on_schema_change` (ADR-0012(d)), dispatch the strategy macro
18
+ (the committed Strategy pattern, ADR-0012(b)) in statement('main'), then drop the temp view.
19
+ Exception (ADR-0029): `delete+insert` with a `unique_key` stages into a real TABLE beside
20
+ the target, `<database>.<schema>.<model>__dbt_tmp_<hash>` (the staging view's name, qualified),
21
+ dropped after statement('main'). Its DELETE and INSERT are two
22
+ statements, and a lazy view would be evaluated again for the INSERT, after the DELETE; a model
23
+ whose `is_incremental()` filter reads `{{ this }}` then staged different rows (DB-700).
24
+
25
+ Deliberate EXCLUSIONS from the stock block (recorded in ADR-0012(a), auditable here):
26
+
27
+ * `adapter.expand_target_column_types` — the base implements it via `alter column ... type`,
28
+ which the Iceberg catalog REJECTS for widening (I6; dbt-spark also omits it). Plan §13.5.
29
+ * `create_indexes` — a default no-op with no Wherobots analog.
30
+
31
+ Adapter-agnostic plumbing preserved verbatim: run_hooks pre/post (inside + outside transaction),
32
+ persist_docs, apply_grants, adapter.commit() (an honest no-op — Spark has no transactions),
33
+ contract handling (carried by `get_create_table_as_sql` on the build paths; the update path's
34
+ only contract use in the stock block gates the excluded `expand_target_column_types`), and
35
+ return({'relations': [target_relation]}).
36
+ #}
37
+ {% materialization incremental, adapter='wherobots' %}
38
+
39
+ {%- set existing_relation = load_cached_relation(this) -%}
40
+ {%- set target_relation = this.incorporate(type='table') -%}
41
+ {%- set temp_relation = make_temp_relation(target_relation) -%}
42
+
43
+ {%- set unique_key = config.get('unique_key') -%}
44
+ {%- set is_view_swap = existing_relation is not none and existing_relation.is_view -%}
45
+ {%- set full_refresh_mode = should_full_refresh() or is_view_swap -%}
46
+ {#- Validated UP FRONT (plan §13.6): our unprefixed `incremental_validate_on_schema_change`
47
+ hard-errors on `sync_all_columns` BEFORE any statement reaches the wire. -#}
48
+ {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}
49
+ {%- set grant_config = config.get('grants') -%}
50
+ {%- set staged = false -%}
51
+
52
+ {{ run_hooks(pre_hooks, inside_transaction=False) }}
53
+ {{ run_hooks(pre_hooks, inside_transaction=True) }}
54
+
55
+ {#- Resolved BEFORE branching (the stock placement): an unsupported builtin strategy
56
+ (`microbatch` — the only omission from valid_incremental_strategies, ADR-0012(b)/0014(e))
57
+ fails here with dbt-core's own clean "not valid for this adapter" error, even on a first
58
+ build, before any DDL/DML is emitted. -#}
59
+ {%- set incremental_strategy = config.get('incremental_strategy') or 'default' -%}
60
+ {%- set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) -%}
61
+
62
+ {#- ADR-0029: `delete+insert` with a `unique_key` runs a DELETE and then an INSERT, so its
63
+ staged rows are materialized into a table first; the INSERT then reads what the DELETE read,
64
+ even when the model's filter reads `{{ this }}`. It takes the staging view's hashed name
65
+ (ADR-0028), qualified beside the target, so it cannot collide with another target's
66
+ staging or replace a user table such as a model named `<model>__dbt_tmp`. Without a
67
+ unique_key the strategy is a single INSERT and keeps the temp view. dbt-adapters also
68
+ resolves the spelling `delete_insert` to the same strategy macro, so both spellings stage. -#}
69
+ {%- set stage_in_table = incremental_strategy in ('delete+insert', 'delete_insert') and unique_key -%}
70
+ {%- if stage_in_table -%}
71
+ {%- set temp_relation = target_relation.incorporate(path={'identifier': temp_relation.identifier}) -%}
72
+ {%- endif -%}
73
+
74
+ {#- ADR-0014(c) — validated UP FRONT, immediately after strategy resolution and BEFORE the
75
+ existing_relation branch (plan §10.3 pinned placement; the Capability-flag stance, the
76
+ `incremental_validate_on_schema_change` precedent): `insert_overwrite` without a USABLE
77
+ `partition_by` hard-errors on the FIRST build too, before any DDL/DML — under the
78
+ engine-default STATIC mode a partition_by-less overwrite silently replaces the WHOLE table
79
+ (P6), and the unpartitioned-replace niche is already served by the atomic CORTAS.
80
+ Truthiness per §10.8: an empty string/list is unset; non-str/list shapes (the dbt-bigquery
81
+ dict habit) are rejected with the same error. -#}
82
+ {%- if incremental_strategy == 'insert_overwrite' -%}
83
+ {%- set partition_by_config = config.get('partition_by') -%}
84
+ {%- set partition_by_usable = false -%}
85
+ {%- if partition_by_config is string -%}
86
+ {%- set partition_by_usable = (partition_by_config | trim) != '' -%}
87
+ {%- elif partition_by_config is not mapping and partition_by_config is sequence -%}
88
+ {#- Review pad 138 finding 2: validate the ENTRIES too — a non-string or empty entry
89
+ would otherwise surface as an opaque Jinja/engine error downstream. -#}
90
+ {%- set usable = namespace(value=(partition_by_config | length) > 0) -%}
91
+ {%- for entry in partition_by_config -%}
92
+ {%- if entry is not string or (entry | trim) == '' -%}
93
+ {%- set usable.value = false -%}
94
+ {%- endif -%}
95
+ {%- endfor -%}
96
+ {%- set partition_by_usable = usable.value -%}
97
+ {%- endif -%}
98
+ {%- if not partition_by_usable -%}
99
+ {%- set partition_by_msg -%}
100
+ incremental_strategy='insert_overwrite' requires a non-empty partition_by (a string or a list of
101
+ strings) on {{ this }} — without it a dynamic overwrite would replace the WHOLE table. For
102
+ whole-table replacement use materialized='table' (or --full-refresh) instead. Note: adopting
103
+ insert_overwrite on an existing table, or changing partition_by, requires --full-refresh
104
+ (ADR-0014).
105
+ {%- endset -%}
106
+ {%- do exceptions.raise_compiler_error(partition_by_msg) -%}
107
+ {%- endif -%}
108
+ {%- endif -%}
109
+
110
+ {% if existing_relation is none %}
111
+ {%- set build_sql = get_create_table_as_sql(False, target_relation, sql) -%}
112
+ {% elif full_refresh_mode %}
113
+ {%- if is_view_swap -%}
114
+ {#- §13.1 / ADR-0010 V7: a cross-type create-or-replace over a VIEW errors — pre-drop the
115
+ squatting view (the type-aware drop emits `drop view if exists <fq>`), THEN create. -#}
116
+ {{ drop_relation_if_exists(existing_relation) }}
117
+ {%- endif -%}
118
+ {%- set build_sql = get_create_table_as_sql(False, target_relation, sql) -%}
119
+ {% else %}
120
+ {#- Stage into the session-scoped temporary view (ADR-0012(c)): `create or replace temporary
121
+ view` is idempotent against same-session leftovers (§13.8). `delete+insert` with a
122
+ unique_key stages into a table instead (ADR-0029); `create or replace table` is
123
+ likewise idempotent against a leftover from a failed run. -#}
124
+ {%- if stage_in_table -%}
125
+ {#- Never partitioned: the staged rows may lack a partition column the target keeps (the
126
+ null-filled INSERT supplies it, ADR-0013). -#}
127
+ {%- set sql_header = config.get('sql_header', none) -%}
128
+ {%- do run_query((sql_header ~ '\n' if sql_header is not none else '') ~ create_or_replace_table_as(temp_relation, sql, none)) -%}
129
+ {%- else -%}
130
+ {%- do run_query(get_create_table_as_sql(True, temp_relation, sql)) -%}
131
+ {%- endif -%}
132
+ {%- set staged = true -%}
133
+ {%- set dest_columns = process_schema_changes(on_schema_change, temp_relation, existing_relation) -%}
134
+ {%- if not dest_columns -%}
135
+ {%- set dest_columns = adapter.get_columns_in_relation(existing_relation) -%}
136
+ {%- endif -%}
137
+ {%- set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) -%}
138
+ {%- set strategy_arg_dict = ({'target_relation': target_relation, 'temp_relation': temp_relation, 'unique_key': unique_key, 'dest_columns': dest_columns, 'incremental_predicates': incremental_predicates}) -%}
139
+ {%- set build_sql = strategy_sql_macro_func(strategy_arg_dict) -%}
140
+ {% endif %}
141
+
142
+ {% call statement("main") %}
143
+ {{ build_sql }}
144
+ {% endcall %}
145
+
146
+ {% if staged %}
147
+ {#- Post-strategy cleanup (ADR-0012(c)): drop the temp view once the strategy statement ran.
148
+ If the strategy statement raises, the materialization aborts before this line — the
149
+ session-scoped self-cleaning of temporary views is the backstop (§13.8), and the
150
+ create-or-replace staging above tolerates any leftover on the next run. A staging table
151
+ (ADR-0029) is dropped the same way; a leftover from a failed run is replaced by the
152
+ next run's `create or replace table`. -#}
153
+ {%- if stage_in_table -%}
154
+ {%- do run_query("drop table if exists " ~ temp_relation) -%}
155
+ {%- else -%}
156
+ {%- do run_query("drop view if exists " ~ temp_relation) -%}
157
+ {%- endif -%}
158
+ {% endif %}
159
+
160
+ {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}
161
+ {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}
162
+
163
+ {% do persist_docs(target_relation, model) %}
164
+
165
+ {{ run_hooks(post_hooks, inside_transaction=True) }}
166
+
167
+ {% do adapter.commit() %}
168
+
169
+ {{ run_hooks(post_hooks, inside_transaction=False) }}
170
+
171
+ {{ return({'relations': [target_relation]}) }}
172
+
173
+ {% endmaterialization %}
@@ -0,0 +1,38 @@
1
+ {#
2
+ incremental_validate_on_schema_change — UNPREFIXED on purpose (plan §13.6 / ADR-0012(d)).
3
+
4
+ The stock macro (`on_schema_change.sql`) is called DIRECTLY by the incremental materialization,
5
+ never through `adapter.dispatch` — so a `wherobots__` prefix here would be dead code (the
6
+ `wherobots__*` convention is reserved for dispatch overrides). Defining the same-named macro in
7
+ this package shadows the global_project copy for our hand-rolled caller.
8
+
9
+ Capability flag, not NotImplementedError (banned), not a silent downgrade: `sync_all_columns` is
10
+ declared UNSUPPORTED with a clear domain error — its type-sync step calls `alter column ... type`,
11
+ which the Iceberg catalog rejects for widening (probe fact I6). The stock behavior is preserved
12
+ for everything else: `ignore` / `fail` / `append_new_columns` pass through, and an UNKNOWN value
13
+ logs and silently downgrades to the default — exactly the stock contract.
14
+ #}
15
+ {% macro incremental_validate_on_schema_change(on_schema_change, default='ignore') %}
16
+
17
+ {%- if on_schema_change == 'sync_all_columns' -%}
18
+
19
+ {%- set msg -%}
20
+ on_schema_change='sync_all_columns' is unsupported on the Wherobots adapter: syncing column
21
+ types requires `alter column ... type`, which the Iceberg catalog rejects (ADR-0012(d)).
22
+ Use 'append_new_columns', 'fail', or 'ignore' instead.
23
+ {%- endset -%}
24
+ {{ exceptions.raise_compiler_error(msg) }}
25
+
26
+ {%- elif on_schema_change not in ['append_new_columns', 'fail', 'ignore'] -%}
27
+
28
+ {%- set log_message = 'Invalid value for on_schema_change (%s) specified. Setting default value of %s.' % (on_schema_change, default) -%}
29
+ {% do log(log_message) %}
30
+ {{ return(default) }}
31
+
32
+ {%- else -%}
33
+
34
+ {{ return(on_schema_change) }}
35
+
36
+ {%- endif -%}
37
+
38
+ {% endmacro %}