dbt-sqlserver 1.10.1rc1__tar.gz → 1.11.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/PKG-INFO +119 -6
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/README.md +116 -3
- dbt_sqlserver-1.11.0/dbt/adapters/sqlserver/__version__.py +1 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/adapters/sqlserver/relation_configs/policies.py +1 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/adapters/sqlserver/sqlserver_adapter.py +113 -9
- dbt_sqlserver-1.11.0/dbt/adapters/sqlserver/sqlserver_configs.py +27 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/adapters/sqlserver/sqlserver_connections.py +118 -11
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/adapters/sqlserver/sqlserver_credentials.py +2 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/adapters/sqlserver/sqlserver_helpers.py +8 -2
- dbt_sqlserver-1.11.0/dbt/adapters/sqlserver/sqlserver_mask.py +218 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/adapters/sqlserver/sqlserver_relation.py +2 -4
- dbt_sqlserver-1.11.0/dbt/include/sqlserver/macros/adapters/apply_masks.sql +170 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/include/sqlserver/macros/adapters/indexes.sql +153 -48
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/include/sqlserver/macros/adapters/metadata.sql +2 -2
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/include/sqlserver/macros/adapters/relation.sql +10 -4
- dbt_sqlserver-1.11.0/dbt/include/sqlserver/macros/materializations/functions/helpers.sql +19 -0
- dbt_sqlserver-1.11.0/dbt/include/sqlserver/macros/materializations/functions/scalar.sql +42 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/include/sqlserver/macros/materializations/hooks.sql +5 -1
- dbt_sqlserver-1.11.0/dbt/include/sqlserver/macros/materializations/models/incremental/incremental.sql +196 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/include/sqlserver/macros/materializations/models/incremental/merge.sql +6 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/include/sqlserver/macros/materializations/models/table/table.sql +49 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/include/sqlserver/macros/materializations/models/table/table_dml_refresh.sql +31 -2
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/include/sqlserver/macros/materializations/snapshots/snapshot.sql +7 -1
- dbt_sqlserver-1.11.0/dbt/include/sqlserver/macros/relations/table/create.sql +264 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt_sqlserver.egg-info/PKG-INFO +119 -6
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt_sqlserver.egg-info/SOURCES.txt +4 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt_sqlserver.egg-info/requires.txt +2 -2
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/pyproject.toml +5 -4
- dbt_sqlserver-1.10.1rc1/dbt/adapters/sqlserver/__version__.py +0 -1
- dbt_sqlserver-1.10.1rc1/dbt/adapters/sqlserver/sqlserver_configs.py +0 -16
- dbt_sqlserver-1.10.1rc1/dbt/include/sqlserver/macros/materializations/models/incremental/incremental.sql +0 -102
- dbt_sqlserver-1.10.1rc1/dbt/include/sqlserver/macros/relations/table/create.sql +0 -50
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/LICENSE +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/__init__.py +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/adapters/sqlserver/__init__.py +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/adapters/sqlserver/py.typed +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/adapters/sqlserver/relation_configs/__init__.py +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/adapters/sqlserver/relation_configs/index.py +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/adapters/sqlserver/sqlserver_auth.py +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/adapters/sqlserver/sqlserver_backend.py +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/adapters/sqlserver/sqlserver_column.py +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/adapters/sqlserver/sqlserver_constants.py +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/adapters/sqlserver/sqlserver_runtime.py +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/include/sqlserver/__init__.py +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/include/sqlserver/dbt_project.yml +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/include/sqlserver/macros/adapters/apply_grants.sql +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/include/sqlserver/macros/adapters/catalog.sql +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/include/sqlserver/macros/adapters/columns.sql +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/include/sqlserver/macros/adapters/persist_docs.sql +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/include/sqlserver/macros/adapters/schema.sql +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/include/sqlserver/macros/adapters/show.sql +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/include/sqlserver/macros/adapters/validate_sql.sql +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/include/sqlserver/macros/materializations/models/incremental/incremental_strategies.sql +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/include/sqlserver/macros/materializations/models/table/columns_spec_ddl.sql +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/include/sqlserver/macros/materializations/models/unit_test/get_fixture_sql.sql +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/include/sqlserver/macros/materializations/models/unit_test/unit_test_create_table_as.sql +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/include/sqlserver/macros/materializations/models/view/create_view_as.sql +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/include/sqlserver/macros/materializations/models/view/view.sql +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/include/sqlserver/macros/materializations/snapshots/helpers.sql +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/include/sqlserver/macros/materializations/snapshots/snapshot_merge.sql +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/include/sqlserver/macros/materializations/snapshots/strategies.sql +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/include/sqlserver/macros/materializations/tests.sql +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/include/sqlserver/macros/materializations/unit_tests.sql +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/include/sqlserver/macros/relations/seeds/helpers.sql +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/include/sqlserver/macros/relations/table/clone.sql +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/include/sqlserver/macros/relations/views/create.sql +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/include/sqlserver/macros/utils/any_value.sql +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/include/sqlserver/macros/utils/array_construct.sql +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/include/sqlserver/macros/utils/cast_bool_to_text.sql +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/include/sqlserver/macros/utils/concat.sql +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/include/sqlserver/macros/utils/date_trunc.sql +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/include/sqlserver/macros/utils/dateadd.sql +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/include/sqlserver/macros/utils/get_tables_by_pattern.sql +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/include/sqlserver/macros/utils/hash.sql +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/include/sqlserver/macros/utils/last_day.sql +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/include/sqlserver/macros/utils/length.sql +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/include/sqlserver/macros/utils/listagg.sql +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/include/sqlserver/macros/utils/position.sql +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/include/sqlserver/macros/utils/safe_cast.sql +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/include/sqlserver/macros/utils/split_part.sql +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/include/sqlserver/macros/utils/timestamps.sql +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/include/sqlserver/profile_template.yml +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt_sqlserver.egg-info/dependency_links.txt +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt_sqlserver.egg-info/top_level.txt +0 -0
- {dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dbt-sqlserver
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.11.0
|
|
4
4
|
Summary: A Microsoft SQL Server adapter plugin for dbt
|
|
5
5
|
Author: Mikael Ene, Anders Swanson, Sam Debruyn, Cor Zuurmond, Cody Scott
|
|
6
6
|
License: MIT
|
|
@@ -21,9 +21,9 @@ Classifier: Programming Language :: Python :: 3.13
|
|
|
21
21
|
Requires-Python: >=3.10
|
|
22
22
|
Description-Content-Type: text/markdown
|
|
23
23
|
License-File: LICENSE
|
|
24
|
-
Requires-Dist: dbt-core<2.0,>=1.
|
|
24
|
+
Requires-Dist: dbt-core<2.0,>=1.11.0
|
|
25
25
|
Requires-Dist: dbt-common<2.0,>=1.22.0
|
|
26
|
-
Requires-Dist: dbt-adapters<2.0,>=1.
|
|
26
|
+
Requires-Dist: dbt-adapters<2.0,>=1.24.1
|
|
27
27
|
Requires-Dist: pyodbc>=5.2.0
|
|
28
28
|
Provides-Extra: azure
|
|
29
29
|
Requires-Dist: azure-identity>=1.12.0; extra == "azure"
|
|
@@ -39,10 +39,34 @@ Dynamic: license-file
|
|
|
39
39
|
|
|
40
40
|
[dbt](https://www.getdbt.com) adapter for Microsoft SQL Server and Azure SQL services.
|
|
41
41
|
|
|
42
|
-
The adapter supports dbt-core
|
|
43
|
-
E.g. version 1.
|
|
42
|
+
The adapter supports dbt-core 1.11 or newer and follows the same versioning scheme.
|
|
43
|
+
E.g. version 1.11.x of the adapter is compatible with dbt-core 1.11.x.
|
|
44
44
|
|
|
45
|
-
|
|
45
|
+
## Supported Python versions
|
|
46
|
+
|
|
47
|
+
The adapter is tested against:
|
|
48
|
+
|
|
49
|
+
| Python version | Status |
|
|
50
|
+
|---|---|
|
|
51
|
+
| 3.10 | Installable (not tested in CI) |
|
|
52
|
+
| 3.11 | Officially supported |
|
|
53
|
+
| 3.12 | Officially supported |
|
|
54
|
+
| 3.13 | Officially supported |
|
|
55
|
+
|
|
56
|
+
## Supported SQL Server versions
|
|
57
|
+
|
|
58
|
+
The adapter is tested against the following SQL Server versions:
|
|
59
|
+
|
|
60
|
+
| SQL Server version | Supported |
|
|
61
|
+
|---|---|
|
|
62
|
+
| SQL Server 2017 | ✅ (minimum supported version) |
|
|
63
|
+
| SQL Server 2019 | ✅ |
|
|
64
|
+
| SQL Server 2022 | ✅ |
|
|
65
|
+
| SQL Server 2025 | ✅ |
|
|
66
|
+
|
|
67
|
+
The minimum supported SQL Server version is SQL Server 2017; older versions are not supported.
|
|
68
|
+
|
|
69
|
+
SQL Server 2017, 2019, 2022, and 2025 are covered by the integration test suite. Azure SQL Database and Azure SQL Managed Instance are not covered by the integration test suite, but are expected to be compatible.
|
|
46
70
|
|
|
47
71
|
## Documentation
|
|
48
72
|
|
|
@@ -195,6 +219,25 @@ flags:
|
|
|
195
219
|
dbt_sqlserver_use_dbt_transactions: true # <-- opt-in; default is false
|
|
196
220
|
```
|
|
197
221
|
|
|
222
|
+
### `xact_abort`
|
|
223
|
+
|
|
224
|
+
*(default: `true`)* Profile/connection field. When enabled, the adapter runs `SET XACT_ABORT ON;` once per connection, right after it opens. With `XACT_ABORT ON`, a run-time error partway through a multi-statement batch (e.g. a `NOT NULL`/constraint violation during the DML table refresh's DELETE+INSERT swap) aborts the whole batch and rolls back any open transaction, instead of only aborting the failing statement and letting a trailing `COMMIT` persist a partial result. See [#718](https://github.com/dbt-msft/dbt-sqlserver/issues/718).
|
|
225
|
+
|
|
226
|
+
This is independent of `dbt_sqlserver_use_dbt_transactions` above: that flag decides who owns the transaction boundary (dbt vs. the driver's autocommit), while `xact_abort` decides how the server reacts to a run-time error mid-batch. `XACT_ABORT ON` matters even when there is no explicit transaction at all, which is exactly the configuration `dbt_sqlserver_use_dbt_transactions` offers no protection in — so the two settings are not derived from one another and both need to be considered independently.
|
|
227
|
+
|
|
228
|
+
Turn it off only if a project intentionally relies on continue-on-error batch semantics (e.g. a hook that expects one failing statement in a batch not to abort the rest):
|
|
229
|
+
|
|
230
|
+
```yaml
|
|
231
|
+
# profiles.yml
|
|
232
|
+
your_profile:
|
|
233
|
+
target: dev
|
|
234
|
+
outputs:
|
|
235
|
+
dev:
|
|
236
|
+
type: sqlserver
|
|
237
|
+
# ...
|
|
238
|
+
xact_abort: false # <-- opt-out; default is true
|
|
239
|
+
```
|
|
240
|
+
|
|
198
241
|
### `column_type_expansion_max_rows`
|
|
199
242
|
|
|
200
243
|
*(default: `1000000`)* Per-model config that limits when safe type expansion runs. When the target table exceeds this row count, safe type expansion is skipped (basic same-family string resizes still proceed). Set to `-1` to disable the check entirely.
|
|
@@ -217,6 +260,76 @@ flags:
|
|
|
217
260
|
|
|
218
261
|
**Compatibility notes:** Enabling `dbt_sqlserver_use_dbt_transactions: true` may expose transaction-state assumptions hidden by autocommit-only mode. Explicit transaction macros may interact with dbt-managed transactions, and cleanup after failed DDL/DML may differ. Review pre/post hooks for in-transaction vs out-of-transaction semantics.
|
|
219
262
|
|
|
263
|
+
### `as_columnstore`
|
|
264
|
+
|
|
265
|
+
*(default: `true`)* When building a table, the adapter creates a [clustered columnstore index](https://learn.microsoft.com/en-us/sql/relational-databases/indexes/columnstore-indexes-overview) (CCI) on it. Set `as_columnstore: false` to build a plain rowstore table instead.
|
|
266
|
+
|
|
267
|
+
This matters for any table containing a `(n)varchar(max)` or other LOB column, because SQL Server does not allow those data types to participate in a columnstore index. The table build fails with:
|
|
268
|
+
|
|
269
|
+
> Column '...' has a data type that cannot participate in a columnstore index.
|
|
270
|
+
|
|
271
|
+
A common case is dbt's [test failure storage](https://docs.getdbt.com/reference/resource-configs/store_failures): the audit tables can contain `VARCHAR(MAX)` columns (dbt's `STRING` type maps to `VARCHAR(MAX)`), so disable the CCI on those resources:
|
|
272
|
+
|
|
273
|
+
```yaml
|
|
274
|
+
# dbt_project.yml
|
|
275
|
+
data_tests:
|
|
276
|
+
+store_failures: true
|
|
277
|
+
+as_columnstore: false # avoids CCI on (n)varchar(max) audit columns
|
|
278
|
+
```
|
|
279
|
+
|
|
280
|
+
You can also set it per model:
|
|
281
|
+
|
|
282
|
+
```sql
|
|
283
|
+
{{ config(materialized="table", as_columnstore=false) }}
|
|
284
|
+
```
|
|
285
|
+
|
|
286
|
+
### Dynamic Data Masking (`masked_with` / `masks`)
|
|
287
|
+
|
|
288
|
+
The adapter can apply SQL Server [Dynamic Data Masking](https://learn.microsoft.com/en-us/sql/relational-databases/security/dynamic-data-masking) (DDM) to columns as part of the materialization, so masks are re-applied on every build and survive dbt's drop-and-recreate on a full refresh. A principal granted `SELECT` but not `UNMASK` then sees masked values instead of real data (dbt's own build principal, being `db_owner`, keeps `UNMASK` and reads real data). Requires **SQL Server 2016+**.
|
|
289
|
+
|
|
290
|
+
There are two config surfaces, and you can use either or both:
|
|
291
|
+
|
|
292
|
+
**Column-level `masked_with:`** — a first-class column property in schema YAML (like `data_type:` or `constraints:`), whose value is the masking-function string:
|
|
293
|
+
|
|
294
|
+
```yaml
|
|
295
|
+
# models/schema.yml
|
|
296
|
+
version: 2
|
|
297
|
+
models:
|
|
298
|
+
- name: core_patients
|
|
299
|
+
columns:
|
|
300
|
+
- name: surname
|
|
301
|
+
masked_with: "default()"
|
|
302
|
+
- name: nhs_number
|
|
303
|
+
masked_with: 'partial(0,"XXXXXXXXXX",0)'
|
|
304
|
+
```
|
|
305
|
+
|
|
306
|
+
**Model-level `masks`** — a `{column: function}` dict, settable in the in-file `{{ config() }}`, the model's `.yml` `config:` block, or a directory-wide default in `dbt_project.yml`. It merges key-wise across those levels (like `meta`), so a directory default and a per-model tweak combine rather than clobber:
|
|
307
|
+
|
|
308
|
+
```sql
|
|
309
|
+
{{ config(masks={'surname': "default()", 'nhs_number': 'partial(0,"XXXXXXXXXX",0)'}) }}
|
|
310
|
+
```
|
|
311
|
+
|
|
312
|
+
```yaml
|
|
313
|
+
# dbt_project.yml — mask nhs_number on every model under datasets/ that has it
|
|
314
|
+
models:
|
|
315
|
+
my_project:
|
|
316
|
+
datasets:
|
|
317
|
+
+masks: { nhs_number: "default()" }
|
|
318
|
+
```
|
|
319
|
+
|
|
320
|
+
Behaviour:
|
|
321
|
+
|
|
322
|
+
- **Precedence:** when both surfaces target the same column, the column-level `masked_with` wins, and a warning naming the model, column and both functions is emitted (even when they agree). This is not something dbt itself ranks, so the rule is the adapter's: a column is more specific than a model.
|
|
323
|
+
- **Opt out of an inherited default:** set `masked_with: null` on the column to remove a mask inherited from a directory/model-level `masks` entry.
|
|
324
|
+
- **Validation:** a `masks` (or `masked_with`) entry naming a column that is not in the built relation is skipped with a warning (a likely typo or stale rename); the run does not fail.
|
|
325
|
+
- **Unmaskable columns:** computed, `FILESTREAM`, sparse `COLUMN_SET` and `Always Encrypted` columns cannot carry a mask, and the run errors listing them rather than emitting DDL that fails.
|
|
326
|
+
- **Views/ephemeral/seeds:** masks apply to base tables only (`table`, `incremental`, `snapshot`). Views inherit masking from their base tables and cannot carry a mask; **seeds are not currently masked**.
|
|
327
|
+
- **Idempotent:** the adapter diffs the desired masks against `sys.masked_columns` and emits only the `ADD` / change / `DROP MASKED` statements that changed, so a persisted (incremental) re-run with no config change issues no DDL.
|
|
328
|
+
|
|
329
|
+
**Indexes and masking.** SQL Server cannot *add* a mask to a column an index depends on (documented for all versions: `ALTER TABLE ALTER COLUMN … failed because one or more objects access this column`). The adapter avoids this on fresh builds by applying masks **before** it creates (rowstore) indexes — which is exactly Microsoft's documented workaround order (mask, then create the index). The default clustered columnstore index is unaffected (its columns are included, not key columns). On a **persisted** table (incremental/snapshot without full refresh), adding a *new* mask to a column that is already an index key errors with a message pointing to the drop-index → mask → recreate-index workaround.
|
|
330
|
+
|
|
331
|
+
**Version notes.** All masking DDL the adapter emits (`ADD MASKED`, `MASKED WITH`, `DROP MASKED`) and the functions `default()`, `email()`, `random(a,b)` and `partial(...)` work on 2016+. The `datetime()` partial-date function and granular column/schema/table-scoped `UNMASK` are SQL Server 2022+ only; the adapter never emits them, but mask-function strings are passed through verbatim, so using a 2022-only function on an older server will be rejected by SQL Server.
|
|
332
|
+
|
|
220
333
|
## Contributing
|
|
221
334
|
|
|
222
335
|
[](https://github.com/dbt-msft/dbt-sqlserver/actions/workflows/unit-tests.yml)
|
|
@@ -2,10 +2,34 @@
|
|
|
2
2
|
|
|
3
3
|
[dbt](https://www.getdbt.com) adapter for Microsoft SQL Server and Azure SQL services.
|
|
4
4
|
|
|
5
|
-
The adapter supports dbt-core
|
|
6
|
-
E.g. version 1.
|
|
5
|
+
The adapter supports dbt-core 1.11 or newer and follows the same versioning scheme.
|
|
6
|
+
E.g. version 1.11.x of the adapter is compatible with dbt-core 1.11.x.
|
|
7
7
|
|
|
8
|
-
|
|
8
|
+
## Supported Python versions
|
|
9
|
+
|
|
10
|
+
The adapter is tested against:
|
|
11
|
+
|
|
12
|
+
| Python version | Status |
|
|
13
|
+
|---|---|
|
|
14
|
+
| 3.10 | Installable (not tested in CI) |
|
|
15
|
+
| 3.11 | Officially supported |
|
|
16
|
+
| 3.12 | Officially supported |
|
|
17
|
+
| 3.13 | Officially supported |
|
|
18
|
+
|
|
19
|
+
## Supported SQL Server versions
|
|
20
|
+
|
|
21
|
+
The adapter is tested against the following SQL Server versions:
|
|
22
|
+
|
|
23
|
+
| SQL Server version | Supported |
|
|
24
|
+
|---|---|
|
|
25
|
+
| SQL Server 2017 | ✅ (minimum supported version) |
|
|
26
|
+
| SQL Server 2019 | ✅ |
|
|
27
|
+
| SQL Server 2022 | ✅ |
|
|
28
|
+
| SQL Server 2025 | ✅ |
|
|
29
|
+
|
|
30
|
+
The minimum supported SQL Server version is SQL Server 2017; older versions are not supported.
|
|
31
|
+
|
|
32
|
+
SQL Server 2017, 2019, 2022, and 2025 are covered by the integration test suite. Azure SQL Database and Azure SQL Managed Instance are not covered by the integration test suite, but are expected to be compatible.
|
|
9
33
|
|
|
10
34
|
## Documentation
|
|
11
35
|
|
|
@@ -158,6 +182,25 @@ flags:
|
|
|
158
182
|
dbt_sqlserver_use_dbt_transactions: true # <-- opt-in; default is false
|
|
159
183
|
```
|
|
160
184
|
|
|
185
|
+
### `xact_abort`
|
|
186
|
+
|
|
187
|
+
*(default: `true`)* Profile/connection field. When enabled, the adapter runs `SET XACT_ABORT ON;` once per connection, right after it opens. With `XACT_ABORT ON`, a run-time error partway through a multi-statement batch (e.g. a `NOT NULL`/constraint violation during the DML table refresh's DELETE+INSERT swap) aborts the whole batch and rolls back any open transaction, instead of only aborting the failing statement and letting a trailing `COMMIT` persist a partial result. See [#718](https://github.com/dbt-msft/dbt-sqlserver/issues/718).
|
|
188
|
+
|
|
189
|
+
This is independent of `dbt_sqlserver_use_dbt_transactions` above: that flag decides who owns the transaction boundary (dbt vs. the driver's autocommit), while `xact_abort` decides how the server reacts to a run-time error mid-batch. `XACT_ABORT ON` matters even when there is no explicit transaction at all, which is exactly the configuration `dbt_sqlserver_use_dbt_transactions` offers no protection in — so the two settings are not derived from one another and both need to be considered independently.
|
|
190
|
+
|
|
191
|
+
Turn it off only if a project intentionally relies on continue-on-error batch semantics (e.g. a hook that expects one failing statement in a batch not to abort the rest):
|
|
192
|
+
|
|
193
|
+
```yaml
|
|
194
|
+
# profiles.yml
|
|
195
|
+
your_profile:
|
|
196
|
+
target: dev
|
|
197
|
+
outputs:
|
|
198
|
+
dev:
|
|
199
|
+
type: sqlserver
|
|
200
|
+
# ...
|
|
201
|
+
xact_abort: false # <-- opt-out; default is true
|
|
202
|
+
```
|
|
203
|
+
|
|
161
204
|
### `column_type_expansion_max_rows`
|
|
162
205
|
|
|
163
206
|
*(default: `1000000`)* Per-model config that limits when safe type expansion runs. When the target table exceeds this row count, safe type expansion is skipped (basic same-family string resizes still proceed). Set to `-1` to disable the check entirely.
|
|
@@ -180,6 +223,76 @@ flags:
|
|
|
180
223
|
|
|
181
224
|
**Compatibility notes:** Enabling `dbt_sqlserver_use_dbt_transactions: true` may expose transaction-state assumptions hidden by autocommit-only mode. Explicit transaction macros may interact with dbt-managed transactions, and cleanup after failed DDL/DML may differ. Review pre/post hooks for in-transaction vs out-of-transaction semantics.
|
|
182
225
|
|
|
226
|
+
### `as_columnstore`
|
|
227
|
+
|
|
228
|
+
*(default: `true`)* When building a table, the adapter creates a [clustered columnstore index](https://learn.microsoft.com/en-us/sql/relational-databases/indexes/columnstore-indexes-overview) (CCI) on it. Set `as_columnstore: false` to build a plain rowstore table instead.
|
|
229
|
+
|
|
230
|
+
This matters for any table containing a `(n)varchar(max)` or other LOB column, because SQL Server does not allow those data types to participate in a columnstore index. The table build fails with:
|
|
231
|
+
|
|
232
|
+
> Column '...' has a data type that cannot participate in a columnstore index.
|
|
233
|
+
|
|
234
|
+
A common case is dbt's [test failure storage](https://docs.getdbt.com/reference/resource-configs/store_failures): the audit tables can contain `VARCHAR(MAX)` columns (dbt's `STRING` type maps to `VARCHAR(MAX)`), so disable the CCI on those resources:
|
|
235
|
+
|
|
236
|
+
```yaml
|
|
237
|
+
# dbt_project.yml
|
|
238
|
+
data_tests:
|
|
239
|
+
+store_failures: true
|
|
240
|
+
+as_columnstore: false # avoids CCI on (n)varchar(max) audit columns
|
|
241
|
+
```
|
|
242
|
+
|
|
243
|
+
You can also set it per model:
|
|
244
|
+
|
|
245
|
+
```sql
|
|
246
|
+
{{ config(materialized="table", as_columnstore=false) }}
|
|
247
|
+
```
|
|
248
|
+
|
|
249
|
+
### Dynamic Data Masking (`masked_with` / `masks`)
|
|
250
|
+
|
|
251
|
+
The adapter can apply SQL Server [Dynamic Data Masking](https://learn.microsoft.com/en-us/sql/relational-databases/security/dynamic-data-masking) (DDM) to columns as part of the materialization, so masks are re-applied on every build and survive dbt's drop-and-recreate on a full refresh. A principal granted `SELECT` but not `UNMASK` then sees masked values instead of real data (dbt's own build principal, being `db_owner`, keeps `UNMASK` and reads real data). Requires **SQL Server 2016+**.
|
|
252
|
+
|
|
253
|
+
There are two config surfaces, and you can use either or both:
|
|
254
|
+
|
|
255
|
+
**Column-level `masked_with:`** — a first-class column property in schema YAML (like `data_type:` or `constraints:`), whose value is the masking-function string:
|
|
256
|
+
|
|
257
|
+
```yaml
|
|
258
|
+
# models/schema.yml
|
|
259
|
+
version: 2
|
|
260
|
+
models:
|
|
261
|
+
- name: core_patients
|
|
262
|
+
columns:
|
|
263
|
+
- name: surname
|
|
264
|
+
masked_with: "default()"
|
|
265
|
+
- name: nhs_number
|
|
266
|
+
masked_with: 'partial(0,"XXXXXXXXXX",0)'
|
|
267
|
+
```
|
|
268
|
+
|
|
269
|
+
**Model-level `masks`** — a `{column: function}` dict, settable in the in-file `{{ config() }}`, the model's `.yml` `config:` block, or a directory-wide default in `dbt_project.yml`. It merges key-wise across those levels (like `meta`), so a directory default and a per-model tweak combine rather than clobber:
|
|
270
|
+
|
|
271
|
+
```sql
|
|
272
|
+
{{ config(masks={'surname': "default()", 'nhs_number': 'partial(0,"XXXXXXXXXX",0)'}) }}
|
|
273
|
+
```
|
|
274
|
+
|
|
275
|
+
```yaml
|
|
276
|
+
# dbt_project.yml — mask nhs_number on every model under datasets/ that has it
|
|
277
|
+
models:
|
|
278
|
+
my_project:
|
|
279
|
+
datasets:
|
|
280
|
+
+masks: { nhs_number: "default()" }
|
|
281
|
+
```
|
|
282
|
+
|
|
283
|
+
Behaviour:
|
|
284
|
+
|
|
285
|
+
- **Precedence:** when both surfaces target the same column, the column-level `masked_with` wins, and a warning naming the model, column and both functions is emitted (even when they agree). This is not something dbt itself ranks, so the rule is the adapter's: a column is more specific than a model.
|
|
286
|
+
- **Opt out of an inherited default:** set `masked_with: null` on the column to remove a mask inherited from a directory/model-level `masks` entry.
|
|
287
|
+
- **Validation:** a `masks` (or `masked_with`) entry naming a column that is not in the built relation is skipped with a warning (a likely typo or stale rename); the run does not fail.
|
|
288
|
+
- **Unmaskable columns:** computed, `FILESTREAM`, sparse `COLUMN_SET` and `Always Encrypted` columns cannot carry a mask, and the run errors listing them rather than emitting DDL that fails.
|
|
289
|
+
- **Views/ephemeral/seeds:** masks apply to base tables only (`table`, `incremental`, `snapshot`). Views inherit masking from their base tables and cannot carry a mask; **seeds are not currently masked**.
|
|
290
|
+
- **Idempotent:** the adapter diffs the desired masks against `sys.masked_columns` and emits only the `ADD` / change / `DROP MASKED` statements that changed, so a persisted (incremental) re-run with no config change issues no DDL.
|
|
291
|
+
|
|
292
|
+
**Indexes and masking.** SQL Server cannot *add* a mask to a column an index depends on (documented for all versions: `ALTER TABLE ALTER COLUMN … failed because one or more objects access this column`). The adapter avoids this on fresh builds by applying masks **before** it creates (rowstore) indexes — which is exactly Microsoft's documented workaround order (mask, then create the index). The default clustered columnstore index is unaffected (its columns are included, not key columns). On a **persisted** table (incremental/snapshot without full refresh), adding a *new* mask to a column that is already an index key errors with a message pointing to the drop-index → mask → recreate-index workaround.
|
|
293
|
+
|
|
294
|
+
**Version notes.** All masking DDL the adapter emits (`ADD MASKED`, `MASKED WITH`, `DROP MASKED`) and the functions `default()`, `email()`, `random(a,b)` and `partial(...)` work on 2016+. The `datetime()` partial-date function and granular column/schema/table-scoped `UNMASK` are SQL Server 2022+ only; the adapter never emits them, but mask-function strings are passed through verbatim, so using a 2022-only function on an older server will be rejected by SQL Server.
|
|
295
|
+
|
|
183
296
|
## Contributing
|
|
184
297
|
|
|
185
298
|
[](https://github.com/dbt-msft/dbt-sqlserver/actions/workflows/unit-tests.yml)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
version = "1.11.0"
|
{dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/adapters/sqlserver/sqlserver_adapter.py
RENAMED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from typing import Any, List, Optional
|
|
1
|
+
from typing import Any, Dict, List, Optional
|
|
2
2
|
|
|
3
3
|
import agate
|
|
4
4
|
import dbt_common.exceptions
|
|
@@ -29,6 +29,9 @@ from dbt.adapters.sqlserver.relation_configs.index import (
|
|
|
29
29
|
from dbt.adapters.sqlserver.sqlserver_column import SQLServerColumn, SQLServerColumnNative
|
|
30
30
|
from dbt.adapters.sqlserver.sqlserver_configs import SQLServerConfigs
|
|
31
31
|
from dbt.adapters.sqlserver.sqlserver_connections import SQLServerConnectionManager
|
|
32
|
+
from dbt.adapters.sqlserver.sqlserver_mask import ColumnMask
|
|
33
|
+
from dbt.adapters.sqlserver.sqlserver_mask import mask_changes as _mask_changes
|
|
34
|
+
from dbt.adapters.sqlserver.sqlserver_mask import resolve_masks as _resolve_masks
|
|
32
35
|
from dbt.adapters.sqlserver.sqlserver_relation import SQLServerRelation
|
|
33
36
|
|
|
34
37
|
logger = AdapterLogger("SQLServer")
|
|
@@ -74,14 +77,6 @@ class SQLServerAdapter(SQLAdapter):
|
|
|
74
77
|
@property
|
|
75
78
|
def _behavior_flags(self) -> List[BehaviorFlag]:
|
|
76
79
|
return [
|
|
77
|
-
{
|
|
78
|
-
"name": "empty",
|
|
79
|
-
"default": False,
|
|
80
|
-
"description": (
|
|
81
|
-
"When enabled, table and view materializations will be created as empty "
|
|
82
|
-
"structures (no data)."
|
|
83
|
-
),
|
|
84
|
-
},
|
|
85
80
|
{
|
|
86
81
|
"name": "dbt_sqlserver_use_default_schema_concat",
|
|
87
82
|
"default": False,
|
|
@@ -424,6 +419,57 @@ class SQLServerAdapter(SQLAdapter):
|
|
|
424
419
|
def parse_index(self, raw_index: Any) -> Optional[SQLServerIndexConfig]:
|
|
425
420
|
return SQLServerIndexConfig.parse(raw_index)
|
|
426
421
|
|
|
422
|
+
@available
|
|
423
|
+
def index_needs_own_batch(self, raw_index: Any) -> bool:
|
|
424
|
+
"""True when raw_index's build_options (ONLINE / RESUMABLE) force its
|
|
425
|
+
CREATE INDEX to run outside any transaction: SQL Server rejects
|
|
426
|
+
RESUMABLE inside a user transaction (error 574), and an ONLINE build
|
|
427
|
+
wrapped in one holds its locks until commit, negating the point."""
|
|
428
|
+
parsed = self.parse_index(raw_index)
|
|
429
|
+
if not parsed:
|
|
430
|
+
return False
|
|
431
|
+
return create_needs_own_batch(parsed.build_options)
|
|
432
|
+
|
|
433
|
+
@available
|
|
434
|
+
def commit_if_open(self) -> None:
|
|
435
|
+
"""Commit the current transaction if one is open - a no-op otherwise.
|
|
436
|
+
|
|
437
|
+
dbt_sqlserver_use_dbt_transactions on (default) wraps a
|
|
438
|
+
materialization's whole build, from its first statement through its
|
|
439
|
+
own trailing ``adapter.commit()``, in one continuous ambient
|
|
440
|
+
transaction. Some statements must not share that transaction with
|
|
441
|
+
whatever runs after them: an ONLINE/RESUMABLE index build (SQL Server
|
|
442
|
+
rejects RESUMABLE inside a user transaction outright, and ONLINE
|
|
443
|
+
holds its locks until commit either way), or a full-refresh-in-
|
|
444
|
+
progress marker, which exists specifically to survive a later
|
|
445
|
+
failure and so must not roll back with it. This makes the prior
|
|
446
|
+
statement durable on its own; pair with begin_if_closed once the
|
|
447
|
+
statement(s) that must run outside a transaction are done, to leave
|
|
448
|
+
later code (more such statements, or the materialization's own
|
|
449
|
+
trailing ``adapter.commit()``, which raises if it finds nothing open)
|
|
450
|
+
working as if this call had never happened.
|
|
451
|
+
|
|
452
|
+
A no-op when no transaction is open: the caller's statement already
|
|
453
|
+
ran autocommitted on its own (e.g. via ``run_query``'s
|
|
454
|
+
``auto_begin=false``, before anything else began one), so there is
|
|
455
|
+
nothing to flush. Also a no-op, at the SQL level, whenever
|
|
456
|
+
dbt_sqlserver_use_dbt_transactions is off: begin/commit still flip
|
|
457
|
+
dbt-core's bookkeeping (see
|
|
458
|
+
SQLServerConnectionManager.add_begin_query/add_commit_query), but
|
|
459
|
+
emit no real T-SQL, matching the driver's own autocommit.
|
|
460
|
+
"""
|
|
461
|
+
connection = self.connections.get_thread_connection()
|
|
462
|
+
if connection is not None and connection.transaction_open:
|
|
463
|
+
self.connections.commit()
|
|
464
|
+
|
|
465
|
+
@available
|
|
466
|
+
def begin_if_closed(self) -> None:
|
|
467
|
+
"""Begin a transaction if none is open - a no-op otherwise. See
|
|
468
|
+
commit_if_open, which this pairs with."""
|
|
469
|
+
connection = self.connections.get_thread_connection()
|
|
470
|
+
if connection is not None and not connection.transaction_open:
|
|
471
|
+
self.connections.begin()
|
|
472
|
+
|
|
427
473
|
@available
|
|
428
474
|
def validate_indexes(
|
|
429
475
|
self, raw_indexes: Any, as_columnstore: Any = False, drop_unmanaged: Any = False
|
|
@@ -502,6 +548,64 @@ class SQLServerAdapter(SQLAdapter):
|
|
|
502
548
|
"warnings": warnings,
|
|
503
549
|
}
|
|
504
550
|
|
|
551
|
+
@available
|
|
552
|
+
def resolve_masks(self, model: Any, model_masks: Optional[dict] = None) -> Dict[str, str]:
|
|
553
|
+
"""Merge the column-level `masked_with` and model-level `masks` surfaces
|
|
554
|
+
into one `{column: function}` map for `apply_masks`.
|
|
555
|
+
|
|
556
|
+
`model` is the Jinja `model` dict (`node.to_dict()`), whose `columns`
|
|
557
|
+
carry any `masked_with` as a flattened key (an explicit `masked_with:
|
|
558
|
+
null` survives serialization as a present `None`, signalling opt-out).
|
|
559
|
+
`model_masks` is `config.get('masks')` — already surface-merged by dbt.
|
|
560
|
+
Precedence and conflict warnings are handled here; key existence is
|
|
561
|
+
validated later in the macro against the real relation.
|
|
562
|
+
"""
|
|
563
|
+
model = model or {}
|
|
564
|
+
columns = model.get("columns") or {}
|
|
565
|
+
column_masks = []
|
|
566
|
+
for name, col in columns.items():
|
|
567
|
+
col = col or {}
|
|
568
|
+
column_masks.append(
|
|
569
|
+
ColumnMask(
|
|
570
|
+
name=col.get("name", name),
|
|
571
|
+
masked_with_present=("masked_with" in col),
|
|
572
|
+
masked_with=col.get("masked_with"),
|
|
573
|
+
)
|
|
574
|
+
)
|
|
575
|
+
model_name = model.get("name") or model.get("alias") or "<unknown>"
|
|
576
|
+
mask_map, warnings = _resolve_masks(column_masks, model_masks, model_name)
|
|
577
|
+
for warning in warnings:
|
|
578
|
+
logger.warning(warning)
|
|
579
|
+
return mask_map
|
|
580
|
+
|
|
581
|
+
@available
|
|
582
|
+
def mask_changes(
|
|
583
|
+
self,
|
|
584
|
+
existing_masks: Any,
|
|
585
|
+
mask_config: Optional[dict],
|
|
586
|
+
index_key_columns: Any = None,
|
|
587
|
+
existing_columns: Any = None,
|
|
588
|
+
) -> dict:
|
|
589
|
+
"""Diff a resolved mask map against current `sys.masked_columns` state.
|
|
590
|
+
|
|
591
|
+
`existing_masks` is the agate table from `get_show_mask_sql` (columns
|
|
592
|
+
`name`, `masking_function`). Returns plain lists for jinja: `adds` /
|
|
593
|
+
`changes` (each `[column, function]`), `drops` (column names), `skipped`
|
|
594
|
+
(warnings for columns absent from the relation) and `errors` (an ADD onto
|
|
595
|
+
a current index-key column, which SQL Server rejects). The macro emits
|
|
596
|
+
DDL for adds/changes/drops, logs `skipped`, and raises on `errors`."""
|
|
597
|
+
rows = []
|
|
598
|
+
if existing_masks is not None:
|
|
599
|
+
column_names = existing_masks.column_names
|
|
600
|
+
for row in existing_masks.rows:
|
|
601
|
+
rows.append(dict(zip(column_names, row)))
|
|
602
|
+
return _mask_changes(
|
|
603
|
+
rows,
|
|
604
|
+
mask_config or {},
|
|
605
|
+
set(index_key_columns or []),
|
|
606
|
+
existing_columns=(list(existing_columns) if existing_columns is not None else None),
|
|
607
|
+
)
|
|
608
|
+
|
|
505
609
|
|
|
506
610
|
COLUMNS_EQUAL_SQL = """
|
|
507
611
|
with diff_count as (
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
from dataclasses import dataclass, field
|
|
2
|
+
from typing import Any, Dict, Optional, Tuple
|
|
3
|
+
|
|
4
|
+
from dbt_common.contracts.config.base import MergeBehavior
|
|
5
|
+
|
|
6
|
+
from dbt.adapters.protocol import AdapterConfig
|
|
7
|
+
from dbt.adapters.sqlserver.relation_configs import SQLServerIndexConfig
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@dataclass
|
|
11
|
+
class SQLServerConfigs(AdapterConfig):
|
|
12
|
+
auto_provision_aad_principals: Optional[bool] = False
|
|
13
|
+
prefer_single_alter_column: Optional[bool] = False
|
|
14
|
+
column_type_expansion_max_rows: int = 1000000
|
|
15
|
+
indexes: Optional[Tuple[SQLServerIndexConfig, ...]] = None
|
|
16
|
+
# false (default) | warn | true - how index reconciliation treats
|
|
17
|
+
# droppable indexes dbt didn't create (YAML may supply bool or str)
|
|
18
|
+
drop_unmanaged_indexes: Optional[Any] = False
|
|
19
|
+
# column-name -> DDM masking-function map for the model-level `masks`
|
|
20
|
+
# surface. MergeBehavior.Update makes it merge key-wise across the config
|
|
21
|
+
# chain (dbt_project.yml +masks defaults, .yml config, in-file config())
|
|
22
|
+
# the same way `meta` composes, rather than the default clobber — so a
|
|
23
|
+
# directory-level default and a per-model tweak combine instead of one
|
|
24
|
+
# replacing the whole dict.
|
|
25
|
+
masks: Optional[Dict[str, Any]] = field(
|
|
26
|
+
default_factory=dict, metadata=MergeBehavior.Update.meta()
|
|
27
|
+
)
|
{dbt_sqlserver-1.10.1rc1 → dbt_sqlserver-1.11.0}/dbt/adapters/sqlserver/sqlserver_connections.py
RENAMED
|
@@ -61,6 +61,14 @@ from dbt.adapters.sqlserver.sqlserver_runtime import (
|
|
|
61
61
|
|
|
62
62
|
logger = AdapterLogger("sqlserver")
|
|
63
63
|
|
|
64
|
+
# Attribute used to stash the in-flight pyodbc / mssql-python cursor on a
|
|
65
|
+
# Connection so cancel() can reach it from another thread. See cancel().
|
|
66
|
+
_IN_FLIGHT_CURSOR_ATTR = "_dbt_sqlserver_in_flight_cursor"
|
|
67
|
+
|
|
68
|
+
# Fires once per process (not per connection/thread) when xact_abort is
|
|
69
|
+
# disabled. See SQLServerConnectionManager._warn_xact_abort_disabled_once.
|
|
70
|
+
_xact_abort_warning_logged = False
|
|
71
|
+
|
|
64
72
|
|
|
65
73
|
class SQLServerConnectionManager(SQLConnectionManager):
|
|
66
74
|
TYPE = "sqlserver"
|
|
@@ -140,10 +148,100 @@ class SQLServerConnectionManager(SQLConnectionManager):
|
|
|
140
148
|
retryable_exceptions=retryable_exceptions,
|
|
141
149
|
)
|
|
142
150
|
|
|
151
|
+
if conn.state == ConnectionState.OPEN:
|
|
152
|
+
if credentials.xact_abort:
|
|
153
|
+
cls._apply_session_settings(conn)
|
|
154
|
+
else:
|
|
155
|
+
cls._warn_xact_abort_disabled_once()
|
|
156
|
+
|
|
143
157
|
return conn
|
|
144
158
|
|
|
145
|
-
|
|
146
|
-
|
|
159
|
+
@classmethod
|
|
160
|
+
def _apply_session_settings(cls, connection: Connection) -> None:
|
|
161
|
+
"""Set session-level defaults that must hold for every batch this
|
|
162
|
+
connection ever runs. Currently: XACT_ABORT, so a run-time error
|
|
163
|
+
mid-batch (e.g. a NOT NULL violation in a DELETE+INSERT swap) kills
|
|
164
|
+
the batch and rolls back any open transaction instead of falling
|
|
165
|
+
through to a trailing COMMIT. See dbt-msft/dbt-sqlserver#718.
|
|
166
|
+
|
|
167
|
+
A connection that fails this setup is not usable: it is closed and
|
|
168
|
+
the error is re-raised rather than handed back to dbt.
|
|
169
|
+
"""
|
|
170
|
+
try:
|
|
171
|
+
cursor = connection.handle.cursor()
|
|
172
|
+
try:
|
|
173
|
+
cursor.execute("SET XACT_ABORT ON;")
|
|
174
|
+
finally:
|
|
175
|
+
cursor.close()
|
|
176
|
+
|
|
177
|
+
if not connection.handle.autocommit:
|
|
178
|
+
connection.handle.commit()
|
|
179
|
+
except Exception:
|
|
180
|
+
connection.handle.close()
|
|
181
|
+
connection.handle = None
|
|
182
|
+
connection.state = ConnectionState.FAIL
|
|
183
|
+
raise
|
|
184
|
+
|
|
185
|
+
@classmethod
|
|
186
|
+
def _warn_xact_abort_disabled_once(cls) -> None:
|
|
187
|
+
global _xact_abort_warning_logged
|
|
188
|
+
if _xact_abort_warning_logged:
|
|
189
|
+
return
|
|
190
|
+
_xact_abort_warning_logged = True
|
|
191
|
+
|
|
192
|
+
use_dbt_transactions = cls._dbt_sqlserver_use_dbt_transactions
|
|
193
|
+
msg = (
|
|
194
|
+
"xact_abort is disabled (xact_abort: false in the profile). Without "
|
|
195
|
+
"SET XACT_ABORT ON, a run-time error partway through a multi-statement "
|
|
196
|
+
"batch (e.g. the DELETE+INSERT swap in the DML refresh materialization) "
|
|
197
|
+
"aborts only the failing statement, not the batch, so a trailing COMMIT "
|
|
198
|
+
"can still commit a partial result. "
|
|
199
|
+
f"dbt_sqlserver_use_dbt_transactions is currently {use_dbt_transactions}"
|
|
200
|
+
)
|
|
201
|
+
if not use_dbt_transactions:
|
|
202
|
+
msg += (
|
|
203
|
+
" (dbt-managed transactions are off): the DML refresh materialization "
|
|
204
|
+
"emits its own in-batch BEGIN/COMMIT, and that in-batch swap can commit "
|
|
205
|
+
"a partial result in this configuration."
|
|
206
|
+
)
|
|
207
|
+
else:
|
|
208
|
+
msg += "."
|
|
209
|
+
logger.warning(msg)
|
|
210
|
+
|
|
211
|
+
def cancel(self, connection: Connection) -> None:
|
|
212
|
+
"""Cancel the in-flight query on ``connection``, if any.
|
|
213
|
+
|
|
214
|
+
dbt-core's ``cancel_open`` calls this for sibling connections when a
|
|
215
|
+
run is interrupted (Ctrl-C) or another thread errors. We cancel by
|
|
216
|
+
calling ``Cursor.cancel()`` on the connection's in-flight cursor:
|
|
217
|
+
pyodbc exposes it and it is explicitly designed to be called from
|
|
218
|
+
another thread (it issues ``SQLCancel``); mssql-python's cursor is
|
|
219
|
+
used the same way when it supports it. Cancellation targets statement
|
|
220
|
+
execution. If no statement is in flight, the cursor is gone, or the
|
|
221
|
+
backend cursor does not support cancellation, this is a best-effort
|
|
222
|
+
no-op.
|
|
223
|
+
"""
|
|
224
|
+
|
|
225
|
+
cursor = getattr(connection, _IN_FLIGHT_CURSOR_ATTR, None)
|
|
226
|
+
if cursor is None:
|
|
227
|
+
logger.debug(f"No in-flight query to cancel for connection {connection.name}.")
|
|
228
|
+
return
|
|
229
|
+
|
|
230
|
+
cancel_cursor = getattr(cursor, "cancel", None)
|
|
231
|
+
if not callable(cancel_cursor):
|
|
232
|
+
logger.debug(
|
|
233
|
+
f"Backend cursor for connection {connection.name} does not "
|
|
234
|
+
"support cancellation; skipping."
|
|
235
|
+
)
|
|
236
|
+
return
|
|
237
|
+
|
|
238
|
+
try:
|
|
239
|
+
logger.debug(f"Cancelling in-flight query for connection {connection.name}.")
|
|
240
|
+
cancel_cursor()
|
|
241
|
+
except Exception as exc:
|
|
242
|
+
# The statement may have completed between the lookup and the
|
|
243
|
+
# cancel; cancellation is best-effort, so swallow and log.
|
|
244
|
+
logger.debug(f"Failed to cancel query for connection {connection.name}: {exc}")
|
|
147
245
|
|
|
148
246
|
def add_begin_query(self):
|
|
149
247
|
if self._dbt_sqlserver_use_dbt_transactions:
|
|
@@ -225,7 +323,7 @@ class SQLServerConnectionManager(SQLConnectionManager):
|
|
|
225
323
|
|
|
226
324
|
fire_event(
|
|
227
325
|
AdapterEventDebug(
|
|
228
|
-
|
|
326
|
+
base_msg=(
|
|
229
327
|
f"Got a retryable error {type(e)}. {retry_limit - attempt} "
|
|
230
328
|
"retries left. Retrying in 1 second.\n"
|
|
231
329
|
f"Error:\n{e}"
|
|
@@ -270,16 +368,25 @@ class SQLServerConnectionManager(SQLConnectionManager):
|
|
|
270
368
|
pre = time.time()
|
|
271
369
|
|
|
272
370
|
cursor = connection.handle.cursor()
|
|
371
|
+
# Track the in-flight cursor so cancel() / cancel_open() can stop it
|
|
372
|
+
# from another thread (e.g. on Ctrl-C); cleared once execution
|
|
373
|
+
# finishes. See cancel().
|
|
374
|
+
setattr(connection, _IN_FLIGHT_CURSOR_ATTR, cursor)
|
|
273
375
|
credentials = self.get_credentials(connection.credentials)
|
|
274
376
|
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
377
|
+
try:
|
|
378
|
+
_execute_query_with_retry(
|
|
379
|
+
cursor=cursor,
|
|
380
|
+
sql=sql,
|
|
381
|
+
bindings=bindings,
|
|
382
|
+
retryable_exceptions=retryable_exceptions,
|
|
383
|
+
# ``retries`` caps total execute attempts, so ``retries: 1``
|
|
384
|
+
# means a single attempt with no retry.
|
|
385
|
+
retry_limit=credentials.retries,
|
|
386
|
+
attempt=1,
|
|
387
|
+
)
|
|
388
|
+
finally:
|
|
389
|
+
setattr(connection, _IN_FLIGHT_CURSOR_ATTR, None)
|
|
283
390
|
|
|
284
391
|
if is_pyodbc_handle(connection.handle):
|
|
285
392
|
connection.handle.add_output_converter(-155, byte_array_to_datetime)
|