etlantic-snowflake 0.38.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- etlantic_snowflake-0.38.0/.gitignore +38 -0
- etlantic_snowflake-0.38.0/PKG-INFO +59 -0
- etlantic_snowflake-0.38.0/README.md +34 -0
- etlantic_snowflake-0.38.0/pyproject.toml +50 -0
- etlantic_snowflake-0.38.0/src/etlantic_snowflake/__init__.py +27 -0
- etlantic_snowflake-0.38.0/src/etlantic_snowflake/connectors.py +417 -0
- etlantic_snowflake-0.38.0/src/etlantic_snowflake/etlantic-plugin-manifest.json +66 -0
- etlantic_snowflake-0.38.0/src/etlantic_snowflake/fake.py +116 -0
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
*.egg-info/
|
|
6
|
+
.eggs/
|
|
7
|
+
dist/
|
|
8
|
+
build/
|
|
9
|
+
site/
|
|
10
|
+
.venv/
|
|
11
|
+
venv/
|
|
12
|
+
.env
|
|
13
|
+
|
|
14
|
+
# Tools
|
|
15
|
+
.pytest_cache/
|
|
16
|
+
.ruff_cache/
|
|
17
|
+
.mypy_cache/
|
|
18
|
+
.coverage
|
|
19
|
+
htmlcov/
|
|
20
|
+
# macOS
|
|
21
|
+
.DS_Store
|
|
22
|
+
**/.DS_Store
|
|
23
|
+
._*
|
|
24
|
+
**/._*
|
|
25
|
+
|
|
26
|
+
examples/_file_storage_out/
|
|
27
|
+
examples/_generated_*.py
|
|
28
|
+
tmp_cli_contracts/
|
|
29
|
+
tmp_/
|
|
30
|
+
|
|
31
|
+
# Local durable workspace (reports, artifacts, schema history)
|
|
32
|
+
.etlantic/
|
|
33
|
+
|
|
34
|
+
# Prefect test home (created by local/CI prefect plugin tests)
|
|
35
|
+
.prefect-test-home/
|
|
36
|
+
|
|
37
|
+
# Local authoring artifacts (not part of the repo)
|
|
38
|
+
pipeline.definition.json
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: etlantic-snowflake
|
|
3
|
+
Version: 0.38.0
|
|
4
|
+
Summary: Experimental Snowflake connector for ETLantic.
|
|
5
|
+
Project-URL: Homepage, https://github.com/eddiethedean/etlantic
|
|
6
|
+
Project-URL: Documentation, https://github.com/eddiethedean/etlantic/tree/main/docs
|
|
7
|
+
Project-URL: Repository, https://github.com/eddiethedean/etlantic
|
|
8
|
+
Project-URL: Issues, https://github.com/eddiethedean/etlantic/issues
|
|
9
|
+
Project-URL: Changelog, https://github.com/eddiethedean/etlantic/blob/main/CHANGELOG.md
|
|
10
|
+
Author-email: Odo Matthews <odosmatthews@gmail.com>
|
|
11
|
+
License-Expression: MIT
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Typing :: Typed
|
|
20
|
+
Requires-Python: >=3.11
|
|
21
|
+
Requires-Dist: etlantic<0.39,>=0.38.0
|
|
22
|
+
Provides-Extra: snowflake
|
|
23
|
+
Requires-Dist: snowflake-connector-python<4,>=3.6; extra == 'snowflake'
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
|
|
26
|
+
# etlantic-snowflake (Experimental)
|
|
27
|
+
|
|
28
|
+
Native Snowflake connector for [ETLantic](https://github.com/eddiethedean/etlantic).
|
|
29
|
+
|
|
30
|
+
**Current release:** 0.38.0.
|
|
31
|
+
|
|
32
|
+
**Maturity:** Experimental (Alpha classifier). CI uses an in-memory fake with
|
|
33
|
+
`autocommit=False` transactional semantics and `query_id` evidence on
|
|
34
|
+
`CommitReceipt`. Live Snowflake is optional:
|
|
35
|
+
|
|
36
|
+
```bash
|
|
37
|
+
pip install "etlantic-snowflake[snowflake]==0.38.0"
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
## Install
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
pip install 'etlantic-snowflake==0.38.0'
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
Core dependency: `etlantic>=0.38.0,<0.39`.
|
|
47
|
+
|
|
48
|
+
## Entry points
|
|
49
|
+
|
|
50
|
+
| Group | Name | Factory |
|
|
51
|
+
|---|---|---|
|
|
52
|
+
| `etlantic.source_connectors` | `snowflake` | `etlantic_snowflake:create_source` |
|
|
53
|
+
| `etlantic.sink_connectors` | `snowflake` | `etlantic_snowflake:create_sink` |
|
|
54
|
+
| `etlantic.storage_connectors` | `snowflake` | `etlantic_snowflake:create_storage` |
|
|
55
|
+
|
|
56
|
+
## Links
|
|
57
|
+
|
|
58
|
+
[Source](https://github.com/eddiethedean/etlantic/tree/main/packages/etlantic-snowflake) ·
|
|
59
|
+
[Issues](https://github.com/eddiethedean/etlantic/issues)
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# etlantic-snowflake (Experimental)
|
|
2
|
+
|
|
3
|
+
Native Snowflake connector for [ETLantic](https://github.com/eddiethedean/etlantic).
|
|
4
|
+
|
|
5
|
+
**Current release:** 0.38.0.
|
|
6
|
+
|
|
7
|
+
**Maturity:** Experimental (Alpha classifier). CI uses an in-memory fake with
|
|
8
|
+
`autocommit=False` transactional semantics and `query_id` evidence on
|
|
9
|
+
`CommitReceipt`. Live Snowflake is optional:
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
pip install "etlantic-snowflake[snowflake]==0.38.0"
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
## Install
|
|
16
|
+
|
|
17
|
+
```bash
|
|
18
|
+
pip install 'etlantic-snowflake==0.38.0'
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
Core dependency: `etlantic>=0.38.0,<0.39`.
|
|
22
|
+
|
|
23
|
+
## Entry points
|
|
24
|
+
|
|
25
|
+
| Group | Name | Factory |
|
|
26
|
+
|---|---|---|
|
|
27
|
+
| `etlantic.source_connectors` | `snowflake` | `etlantic_snowflake:create_source` |
|
|
28
|
+
| `etlantic.sink_connectors` | `snowflake` | `etlantic_snowflake:create_sink` |
|
|
29
|
+
| `etlantic.storage_connectors` | `snowflake` | `etlantic_snowflake:create_storage` |
|
|
30
|
+
|
|
31
|
+
## Links
|
|
32
|
+
|
|
33
|
+
[Source](https://github.com/eddiethedean/etlantic/tree/main/packages/etlantic-snowflake) ·
|
|
34
|
+
[Issues](https://github.com/eddiethedean/etlantic/issues)
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "etlantic-snowflake"
|
|
3
|
+
version = "0.38.0"
|
|
4
|
+
description = "Experimental Snowflake connector for ETLantic."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
requires-python = ">=3.11"
|
|
8
|
+
authors = [{ name = "Odo Matthews", email = "odosmatthews@gmail.com" }]
|
|
9
|
+
classifiers = [
|
|
10
|
+
"Development Status :: 3 - Alpha",
|
|
11
|
+
"Intended Audience :: Developers",
|
|
12
|
+
"License :: OSI Approved :: MIT License",
|
|
13
|
+
"Programming Language :: Python :: 3",
|
|
14
|
+
"Programming Language :: Python :: 3.11",
|
|
15
|
+
"Programming Language :: Python :: 3.12",
|
|
16
|
+
"Programming Language :: Python :: 3.13",
|
|
17
|
+
"Typing :: Typed",
|
|
18
|
+
]
|
|
19
|
+
dependencies = [
|
|
20
|
+
"etlantic>=0.38.0,<0.39",
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
[project.optional-dependencies]
|
|
24
|
+
snowflake = ["snowflake-connector-python>=3.6,<4"]
|
|
25
|
+
|
|
26
|
+
[project.urls]
|
|
27
|
+
Homepage = "https://github.com/eddiethedean/etlantic"
|
|
28
|
+
Documentation = "https://github.com/eddiethedean/etlantic/tree/main/docs"
|
|
29
|
+
Repository = "https://github.com/eddiethedean/etlantic"
|
|
30
|
+
Issues = "https://github.com/eddiethedean/etlantic/issues"
|
|
31
|
+
Changelog = "https://github.com/eddiethedean/etlantic/blob/main/CHANGELOG.md"
|
|
32
|
+
|
|
33
|
+
[project.entry-points."etlantic.source_connectors"]
|
|
34
|
+
snowflake = "etlantic_snowflake:create_source"
|
|
35
|
+
|
|
36
|
+
[project.entry-points."etlantic.sink_connectors"]
|
|
37
|
+
snowflake = "etlantic_snowflake:create_sink"
|
|
38
|
+
|
|
39
|
+
[project.entry-points."etlantic.storage_connectors"]
|
|
40
|
+
snowflake = "etlantic_snowflake:create_storage"
|
|
41
|
+
|
|
42
|
+
[build-system]
|
|
43
|
+
requires = ["hatchling"]
|
|
44
|
+
build-backend = "hatchling.build"
|
|
45
|
+
|
|
46
|
+
[tool.hatch.build.targets.wheel]
|
|
47
|
+
packages = ["src/etlantic_snowflake"]
|
|
48
|
+
|
|
49
|
+
[tool.hatch.build.targets.wheel.force-include]
|
|
50
|
+
"src/etlantic_snowflake/etlantic-plugin-manifest.json" = "etlantic_snowflake/etlantic-plugin-manifest.json"
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"""Experimental Snowflake connector package for ETLantic."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
__version__ = "0.38.0"
|
|
6
|
+
|
|
7
|
+
from etlantic_snowflake.connectors import (
|
|
8
|
+
SnowflakeSinkConnector,
|
|
9
|
+
SnowflakeSourceConnector,
|
|
10
|
+
SnowflakeStorageConnector,
|
|
11
|
+
create_sink,
|
|
12
|
+
create_source,
|
|
13
|
+
create_storage,
|
|
14
|
+
)
|
|
15
|
+
from etlantic_snowflake.fake import FakeSnowflakeConnection, snowflake_sdk_available
|
|
16
|
+
|
|
17
|
+
__all__ = [
|
|
18
|
+
"FakeSnowflakeConnection",
|
|
19
|
+
"SnowflakeSinkConnector",
|
|
20
|
+
"SnowflakeSourceConnector",
|
|
21
|
+
"SnowflakeStorageConnector",
|
|
22
|
+
"__version__",
|
|
23
|
+
"create_sink",
|
|
24
|
+
"create_source",
|
|
25
|
+
"create_storage",
|
|
26
|
+
"snowflake_sdk_available",
|
|
27
|
+
]
|
|
@@ -0,0 +1,417 @@
|
|
|
1
|
+
"""Snowflake source / sink / storage connectors (fake by default)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import uuid
|
|
6
|
+
from collections.abc import AsyncIterator, Mapping
|
|
7
|
+
from dataclasses import dataclass, field
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
from etlantic.connectors.capabilities import (
|
|
11
|
+
IDEMPOTENCY,
|
|
12
|
+
PUBLICATION_ATOMIC,
|
|
13
|
+
RECONCILIATION,
|
|
14
|
+
SOURCE_BATCH_SNAPSHOT,
|
|
15
|
+
SOURCE_SCHEMA_DISCOVERY,
|
|
16
|
+
SOURCE_STATISTICS_BOUNDED,
|
|
17
|
+
TRANSACTIONS,
|
|
18
|
+
WRITE_APPEND,
|
|
19
|
+
WRITE_MERGE,
|
|
20
|
+
WRITE_OVERWRITE,
|
|
21
|
+
)
|
|
22
|
+
from etlantic.connectors.errors import ConnectorConfigError, ConnectorWriteError
|
|
23
|
+
from etlantic.connectors.maturity import ConnectorMaturity
|
|
24
|
+
from etlantic.connectors.models import (
|
|
25
|
+
SINK_PROTOCOL,
|
|
26
|
+
SOURCE_PROTOCOL,
|
|
27
|
+
STORAGE_PROTOCOL,
|
|
28
|
+
CleanupReceipt,
|
|
29
|
+
CommitReceipt,
|
|
30
|
+
ConnectorInfo,
|
|
31
|
+
CursorProposal,
|
|
32
|
+
LandingReadManifest,
|
|
33
|
+
ReadBatch,
|
|
34
|
+
ReconciliationResult,
|
|
35
|
+
SchemaInspection,
|
|
36
|
+
SinkPlan,
|
|
37
|
+
SourcePlan,
|
|
38
|
+
WriteSession,
|
|
39
|
+
fingerprint_public_config,
|
|
40
|
+
)
|
|
41
|
+
from etlantic_snowflake.fake import FakeSnowflakeConnection, snowflake_sdk_available
|
|
42
|
+
|
|
43
|
+
PROVIDER = "snowflake"
|
|
44
|
+
PACKAGE_VERSION = "0.38.0"
|
|
45
|
+
|
|
46
|
+
SOURCE_CAPS = frozenset(
|
|
47
|
+
{
|
|
48
|
+
SOURCE_BATCH_SNAPSHOT,
|
|
49
|
+
SOURCE_SCHEMA_DISCOVERY,
|
|
50
|
+
SOURCE_STATISTICS_BOUNDED,
|
|
51
|
+
IDEMPOTENCY,
|
|
52
|
+
}
|
|
53
|
+
)
|
|
54
|
+
SINK_CAPS = frozenset(
|
|
55
|
+
{
|
|
56
|
+
WRITE_APPEND,
|
|
57
|
+
WRITE_OVERWRITE,
|
|
58
|
+
WRITE_MERGE,
|
|
59
|
+
PUBLICATION_ATOMIC,
|
|
60
|
+
TRANSACTIONS,
|
|
61
|
+
RECONCILIATION,
|
|
62
|
+
IDEMPOTENCY,
|
|
63
|
+
}
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _public_config(binding: Mapping[str, Any]) -> dict[str, Any]:
|
|
68
|
+
raw = binding.get("config")
|
|
69
|
+
if isinstance(raw, dict):
|
|
70
|
+
return dict(raw)
|
|
71
|
+
keys = ("table", "schema", "database", "mode", "warehouse")
|
|
72
|
+
return {k: binding[k] for k in keys if k in binding and binding[k] is not None}
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _table_name(binding: Mapping[str, Any], cfg: Mapping[str, Any]) -> str:
|
|
76
|
+
table = str(cfg.get("table") or binding.get("location") or "")
|
|
77
|
+
if not table:
|
|
78
|
+
raise ConnectorConfigError(
|
|
79
|
+
"snowflake binding requires table",
|
|
80
|
+
code="PMCONN831",
|
|
81
|
+
provider=PROVIDER,
|
|
82
|
+
)
|
|
83
|
+
schema = cfg.get("schema")
|
|
84
|
+
database = cfg.get("database")
|
|
85
|
+
parts = [p for p in (database, schema, table) if p]
|
|
86
|
+
return ".".join(str(p) for p in parts)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
@dataclass
|
|
90
|
+
class SnowflakeSourceConnector:
|
|
91
|
+
connection: FakeSnowflakeConnection = field(default_factory=FakeSnowflakeConnection)
|
|
92
|
+
|
|
93
|
+
def info(self) -> ConnectorInfo:
|
|
94
|
+
return ConnectorInfo(
|
|
95
|
+
name=PROVIDER,
|
|
96
|
+
protocol=SOURCE_PROTOCOL,
|
|
97
|
+
version=PACKAGE_VERSION,
|
|
98
|
+
provider=PROVIDER,
|
|
99
|
+
capabilities=tuple(sorted(SOURCE_CAPS)),
|
|
100
|
+
maturity=ConnectorMaturity.EXPERIMENTAL,
|
|
101
|
+
metadata={
|
|
102
|
+
"fake": not snowflake_sdk_available(),
|
|
103
|
+
"autocommit": False,
|
|
104
|
+
},
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
async def plan_read(
|
|
108
|
+
self,
|
|
109
|
+
*,
|
|
110
|
+
binding: Mapping[str, Any],
|
|
111
|
+
context: Mapping[str, Any],
|
|
112
|
+
) -> SourcePlan:
|
|
113
|
+
cfg = _public_config(binding)
|
|
114
|
+
table = _table_name(binding, cfg)
|
|
115
|
+
return SourcePlan(
|
|
116
|
+
provider=PROVIDER,
|
|
117
|
+
protocol=SOURCE_PROTOCOL,
|
|
118
|
+
mode="snapshot",
|
|
119
|
+
identity_scheme="snowflake_query_id/1",
|
|
120
|
+
listing_intent={"table": table},
|
|
121
|
+
config_fingerprint=fingerprint_public_config(cfg),
|
|
122
|
+
root_ref=table,
|
|
123
|
+
secret_refs=tuple(
|
|
124
|
+
sorted(str(k) for k in (binding.get("secret_refs") or {}))
|
|
125
|
+
),
|
|
126
|
+
)
|
|
127
|
+
|
|
128
|
+
async def read_batches(
|
|
129
|
+
self,
|
|
130
|
+
*,
|
|
131
|
+
plan: SourcePlan,
|
|
132
|
+
binding: Mapping[str, Any],
|
|
133
|
+
context: Mapping[str, Any],
|
|
134
|
+
) -> AsyncIterator[ReadBatch]:
|
|
135
|
+
table = str(plan.listing_intent.get("table") or "")
|
|
136
|
+
result = self.connection.select(table)
|
|
137
|
+
yield ReadBatch(
|
|
138
|
+
records=result.rows,
|
|
139
|
+
batch_index=0,
|
|
140
|
+
exhausted=True,
|
|
141
|
+
metadata={"table": table, "query_id": result.query_id},
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
async def propose_cursor(
|
|
145
|
+
self,
|
|
146
|
+
*,
|
|
147
|
+
plan: SourcePlan,
|
|
148
|
+
manifest: LandingReadManifest,
|
|
149
|
+
context: Mapping[str, Any],
|
|
150
|
+
) -> CursorProposal | None:
|
|
151
|
+
return None
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
@dataclass
|
|
155
|
+
class SnowflakeSinkConnector:
|
|
156
|
+
connection: FakeSnowflakeConnection = field(default_factory=FakeSnowflakeConnection)
|
|
157
|
+
_sessions: dict[str, dict[str, Any]] = field(default_factory=dict, repr=False)
|
|
158
|
+
|
|
159
|
+
def info(self) -> ConnectorInfo:
|
|
160
|
+
return ConnectorInfo(
|
|
161
|
+
name=PROVIDER,
|
|
162
|
+
protocol=SINK_PROTOCOL,
|
|
163
|
+
version=PACKAGE_VERSION,
|
|
164
|
+
provider=PROVIDER,
|
|
165
|
+
capabilities=tuple(sorted(SINK_CAPS)),
|
|
166
|
+
maturity=ConnectorMaturity.EXPERIMENTAL,
|
|
167
|
+
metadata={
|
|
168
|
+
"fake": not snowflake_sdk_available(),
|
|
169
|
+
"autocommit": False,
|
|
170
|
+
"evidence": "query_id",
|
|
171
|
+
},
|
|
172
|
+
)
|
|
173
|
+
|
|
174
|
+
async def plan_write(
|
|
175
|
+
self,
|
|
176
|
+
*,
|
|
177
|
+
binding: Mapping[str, Any],
|
|
178
|
+
context: Mapping[str, Any],
|
|
179
|
+
) -> SinkPlan:
|
|
180
|
+
cfg = _public_config(binding)
|
|
181
|
+
table = _table_name(binding, cfg)
|
|
182
|
+
mode = str(cfg.get("mode") or binding.get("mode") or "append")
|
|
183
|
+
if mode not in {"append", "overwrite", "replace", "merge"}:
|
|
184
|
+
raise ConnectorConfigError(
|
|
185
|
+
f"unsupported snowflake write mode {mode!r}",
|
|
186
|
+
code="PMCONN832",
|
|
187
|
+
provider=PROVIDER,
|
|
188
|
+
)
|
|
189
|
+
return SinkPlan(
|
|
190
|
+
provider=PROVIDER,
|
|
191
|
+
protocol=SINK_PROTOCOL,
|
|
192
|
+
write_mode=mode,
|
|
193
|
+
config_fingerprint=fingerprint_public_config(cfg),
|
|
194
|
+
root_ref=table,
|
|
195
|
+
secret_refs=tuple(
|
|
196
|
+
sorted(str(k) for k in (binding.get("secret_refs") or {}))
|
|
197
|
+
),
|
|
198
|
+
metadata={"table": table},
|
|
199
|
+
)
|
|
200
|
+
|
|
201
|
+
async def begin_write(
|
|
202
|
+
self,
|
|
203
|
+
*,
|
|
204
|
+
plan: SinkPlan,
|
|
205
|
+
binding: Mapping[str, Any],
|
|
206
|
+
context: Mapping[str, Any],
|
|
207
|
+
) -> WriteSession:
|
|
208
|
+
if self.connection.autocommit:
|
|
209
|
+
raise ConnectorWriteError(
|
|
210
|
+
"snowflake transactional path requires autocommit=False",
|
|
211
|
+
code="PMCONN833",
|
|
212
|
+
provider=PROVIDER,
|
|
213
|
+
)
|
|
214
|
+
session_id = f"sf-{uuid.uuid4().hex[:12]}"
|
|
215
|
+
self._sessions[session_id] = {
|
|
216
|
+
"table": str(plan.metadata.get("table") or plan.root_ref),
|
|
217
|
+
"mode": plan.write_mode or "append",
|
|
218
|
+
"rows": [],
|
|
219
|
+
"query_ids": [],
|
|
220
|
+
"status": "open",
|
|
221
|
+
}
|
|
222
|
+
return WriteSession(
|
|
223
|
+
session_id=session_id,
|
|
224
|
+
provider=PROVIDER,
|
|
225
|
+
protocol=SINK_PROTOCOL,
|
|
226
|
+
metadata={"table": plan.metadata.get("table"), "autocommit": False},
|
|
227
|
+
)
|
|
228
|
+
|
|
229
|
+
async def write_batch(
|
|
230
|
+
self,
|
|
231
|
+
session: WriteSession,
|
|
232
|
+
batch: Any,
|
|
233
|
+
*,
|
|
234
|
+
context: Mapping[str, Any],
|
|
235
|
+
) -> None:
|
|
236
|
+
state = self._require(session.session_id)
|
|
237
|
+
if isinstance(batch, Mapping):
|
|
238
|
+
state["rows"].append(dict(batch))
|
|
239
|
+
elif isinstance(batch, (list, tuple)):
|
|
240
|
+
for item in batch:
|
|
241
|
+
state["rows"].append(
|
|
242
|
+
dict(item) if isinstance(item, Mapping) else {"value": item}
|
|
243
|
+
)
|
|
244
|
+
else:
|
|
245
|
+
state["rows"].append({"value": batch})
|
|
246
|
+
|
|
247
|
+
async def prepare(
|
|
248
|
+
self,
|
|
249
|
+
session: WriteSession,
|
|
250
|
+
*,
|
|
251
|
+
context: Mapping[str, Any],
|
|
252
|
+
) -> None:
|
|
253
|
+
state = self._require(session.session_id)
|
|
254
|
+
result = self.connection.execute(
|
|
255
|
+
f"STAGE {state['mode']} INTO {state['table']}",
|
|
256
|
+
table=str(state["table"]),
|
|
257
|
+
rows=list(state["rows"]),
|
|
258
|
+
operation=str(state["mode"]),
|
|
259
|
+
)
|
|
260
|
+
state["query_ids"].append(result.query_id)
|
|
261
|
+
state["prepared"] = True
|
|
262
|
+
|
|
263
|
+
async def commit(
|
|
264
|
+
self,
|
|
265
|
+
session: WriteSession,
|
|
266
|
+
*,
|
|
267
|
+
context: Mapping[str, Any],
|
|
268
|
+
) -> CommitReceipt:
|
|
269
|
+
state = self._require(session.session_id)
|
|
270
|
+
if not state.get("prepared"):
|
|
271
|
+
await self.prepare(session, context=context)
|
|
272
|
+
query_ids = self.connection.commit()
|
|
273
|
+
# Prefer session-staged ids; fall back to connection commit list.
|
|
274
|
+
evidence_ids = list(state["query_ids"]) or query_ids
|
|
275
|
+
primary = evidence_ids[-1] if evidence_ids else None
|
|
276
|
+
state["status"] = "committed"
|
|
277
|
+
return CommitReceipt(
|
|
278
|
+
status="committed",
|
|
279
|
+
session_id=session.session_id,
|
|
280
|
+
provider=PROVIDER,
|
|
281
|
+
publication_id=primary,
|
|
282
|
+
message="snowflake transaction committed",
|
|
283
|
+
metadata={
|
|
284
|
+
"table": state["table"],
|
|
285
|
+
"query_id": primary,
|
|
286
|
+
"query_ids": evidence_ids,
|
|
287
|
+
"autocommit": False,
|
|
288
|
+
},
|
|
289
|
+
)
|
|
290
|
+
|
|
291
|
+
async def abort(
|
|
292
|
+
self,
|
|
293
|
+
session: WriteSession,
|
|
294
|
+
*,
|
|
295
|
+
context: Mapping[str, Any],
|
|
296
|
+
) -> CommitReceipt:
|
|
297
|
+
state = self._require(session.session_id)
|
|
298
|
+
self.connection.rollback()
|
|
299
|
+
state["status"] = "aborted"
|
|
300
|
+
return CommitReceipt(
|
|
301
|
+
status="rolled_back",
|
|
302
|
+
session_id=session.session_id,
|
|
303
|
+
provider=PROVIDER,
|
|
304
|
+
message="snowflake transaction rolled back",
|
|
305
|
+
metadata={
|
|
306
|
+
"table": state["table"],
|
|
307
|
+
"query_ids": list(state["query_ids"]),
|
|
308
|
+
"autocommit": False,
|
|
309
|
+
},
|
|
310
|
+
)
|
|
311
|
+
|
|
312
|
+
async def reconcile(
|
|
313
|
+
self,
|
|
314
|
+
receipt: CommitReceipt,
|
|
315
|
+
*,
|
|
316
|
+
context: Mapping[str, Any],
|
|
317
|
+
) -> ReconciliationResult:
|
|
318
|
+
query_id = receipt.publication_id or (receipt.metadata or {}).get("query_id")
|
|
319
|
+
if not query_id:
|
|
320
|
+
return ReconciliationResult(status="unknown", message="missing query_id")
|
|
321
|
+
found = self.connection.lookup_query(str(query_id))
|
|
322
|
+
if found is None:
|
|
323
|
+
return ReconciliationResult(
|
|
324
|
+
status="rolled_back",
|
|
325
|
+
message="query_id not found",
|
|
326
|
+
metadata={"query_id": query_id},
|
|
327
|
+
)
|
|
328
|
+
if str(query_id) in self.connection._pending_queries:
|
|
329
|
+
return ReconciliationResult(
|
|
330
|
+
status="unknown",
|
|
331
|
+
publication_id=str(query_id),
|
|
332
|
+
message="query still pending in open transaction",
|
|
333
|
+
metadata={"query_id": query_id},
|
|
334
|
+
)
|
|
335
|
+
return ReconciliationResult(
|
|
336
|
+
status="committed",
|
|
337
|
+
publication_id=str(query_id),
|
|
338
|
+
message="query_id found in committed history",
|
|
339
|
+
metadata={"query_id": query_id},
|
|
340
|
+
)
|
|
341
|
+
|
|
342
|
+
async def cleanup(
|
|
343
|
+
self,
|
|
344
|
+
receipt: CommitReceipt,
|
|
345
|
+
*,
|
|
346
|
+
context: Mapping[str, Any],
|
|
347
|
+
) -> CleanupReceipt:
|
|
348
|
+
return CleanupReceipt(status="skipped", message="no cleanup for snowflake fake")
|
|
349
|
+
|
|
350
|
+
def _require(self, session_id: str) -> dict[str, Any]:
|
|
351
|
+
state = self._sessions.get(session_id)
|
|
352
|
+
if state is None:
|
|
353
|
+
raise ConnectorWriteError(
|
|
354
|
+
f"unknown snowflake session {session_id!r}",
|
|
355
|
+
code="PMCONN834",
|
|
356
|
+
provider=PROVIDER,
|
|
357
|
+
)
|
|
358
|
+
return state
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
@dataclass
|
|
362
|
+
class SnowflakeStorageConnector:
|
|
363
|
+
connection: FakeSnowflakeConnection = field(default_factory=FakeSnowflakeConnection)
|
|
364
|
+
|
|
365
|
+
def info(self) -> ConnectorInfo:
|
|
366
|
+
return ConnectorInfo(
|
|
367
|
+
name=PROVIDER,
|
|
368
|
+
protocol=STORAGE_PROTOCOL,
|
|
369
|
+
version=PACKAGE_VERSION,
|
|
370
|
+
provider=PROVIDER,
|
|
371
|
+
capabilities=(SOURCE_SCHEMA_DISCOVERY, SOURCE_STATISTICS_BOUNDED),
|
|
372
|
+
maturity=ConnectorMaturity.EXPERIMENTAL,
|
|
373
|
+
metadata={"fake": not snowflake_sdk_available()},
|
|
374
|
+
)
|
|
375
|
+
|
|
376
|
+
async def inspect_schema(
|
|
377
|
+
self,
|
|
378
|
+
*,
|
|
379
|
+
binding: Mapping[str, Any],
|
|
380
|
+
context: Mapping[str, Any],
|
|
381
|
+
) -> SchemaInspection:
|
|
382
|
+
cfg = _public_config(binding)
|
|
383
|
+
table = _table_name(binding, cfg)
|
|
384
|
+
rows = self.connection.tables.get(table, [])
|
|
385
|
+
fields: tuple[dict[str, Any], ...] = ()
|
|
386
|
+
if rows:
|
|
387
|
+
fields = tuple(
|
|
388
|
+
{"name": k, "type": type(v).__name__} for k, v in rows[0].items()
|
|
389
|
+
)
|
|
390
|
+
return SchemaInspection(
|
|
391
|
+
provider=PROVIDER,
|
|
392
|
+
fields=fields,
|
|
393
|
+
row_estimate=len(rows),
|
|
394
|
+
metadata={"table": table},
|
|
395
|
+
)
|
|
396
|
+
|
|
397
|
+
|
|
398
|
+
def create_source() -> SnowflakeSourceConnector:
|
|
399
|
+
return SnowflakeSourceConnector()
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
def create_sink() -> SnowflakeSinkConnector:
|
|
403
|
+
return SnowflakeSinkConnector()
|
|
404
|
+
|
|
405
|
+
|
|
406
|
+
def create_storage() -> SnowflakeStorageConnector:
|
|
407
|
+
return SnowflakeStorageConnector()
|
|
408
|
+
|
|
409
|
+
|
|
410
|
+
__all__ = [
|
|
411
|
+
"SnowflakeSinkConnector",
|
|
412
|
+
"SnowflakeSourceConnector",
|
|
413
|
+
"SnowflakeStorageConnector",
|
|
414
|
+
"create_sink",
|
|
415
|
+
"create_source",
|
|
416
|
+
"create_storage",
|
|
417
|
+
]
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
{
|
|
2
|
+
"capabilities": [
|
|
3
|
+
"source.batch_snapshot",
|
|
4
|
+
"source.schema_discovery",
|
|
5
|
+
"source.statistics_bounded",
|
|
6
|
+
"write.append",
|
|
7
|
+
"write.overwrite",
|
|
8
|
+
"write.merge",
|
|
9
|
+
"publication.atomic",
|
|
10
|
+
"transactions",
|
|
11
|
+
"reconciliation",
|
|
12
|
+
"idempotency"
|
|
13
|
+
],
|
|
14
|
+
"digest": "sha256:3f13efa403ec55103580cc0794e4b321dd073021c23967c2533b87e5c9992385",
|
|
15
|
+
"entries": [
|
|
16
|
+
{
|
|
17
|
+
"capabilities": [
|
|
18
|
+
"source.batch_snapshot",
|
|
19
|
+
"source.schema_discovery",
|
|
20
|
+
"source.statistics_bounded",
|
|
21
|
+
"idempotency"
|
|
22
|
+
],
|
|
23
|
+
"engine": "snowflake",
|
|
24
|
+
"group": "etlantic.source_connectors",
|
|
25
|
+
"name": "snowflake",
|
|
26
|
+
"protocol": "etlantic.source/1",
|
|
27
|
+
"target": "etlantic_snowflake:create_source"
|
|
28
|
+
},
|
|
29
|
+
{
|
|
30
|
+
"capabilities": [
|
|
31
|
+
"write.append",
|
|
32
|
+
"write.overwrite",
|
|
33
|
+
"write.merge",
|
|
34
|
+
"publication.atomic",
|
|
35
|
+
"transactions",
|
|
36
|
+
"reconciliation",
|
|
37
|
+
"idempotency"
|
|
38
|
+
],
|
|
39
|
+
"engine": "snowflake",
|
|
40
|
+
"group": "etlantic.sink_connectors",
|
|
41
|
+
"name": "snowflake",
|
|
42
|
+
"protocol": "etlantic.sink/1",
|
|
43
|
+
"target": "etlantic_snowflake:create_sink"
|
|
44
|
+
},
|
|
45
|
+
{
|
|
46
|
+
"capabilities": [
|
|
47
|
+
"source.schema_discovery",
|
|
48
|
+
"source.statistics_bounded"
|
|
49
|
+
],
|
|
50
|
+
"engine": "snowflake",
|
|
51
|
+
"group": "etlantic.storage_connectors",
|
|
52
|
+
"name": "snowflake",
|
|
53
|
+
"protocol": "etlantic.storage/1",
|
|
54
|
+
"target": "etlantic_snowflake:create_storage"
|
|
55
|
+
}
|
|
56
|
+
],
|
|
57
|
+
"package": "etlantic-snowflake",
|
|
58
|
+
"privileges": [],
|
|
59
|
+
"protocol_range": "etlantic.source/1,etlantic.sink/1,etlantic.storage/1",
|
|
60
|
+
"provenance": {
|
|
61
|
+
"publisher": "etlantic",
|
|
62
|
+
"maturity": "experimental"
|
|
63
|
+
},
|
|
64
|
+
"schema": "etlantic.plugin_manifest/1",
|
|
65
|
+
"version": "0.38.0"
|
|
66
|
+
}
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
"""In-memory Snowflake fake: autocommit-off + query_id evidence."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import uuid
|
|
6
|
+
from dataclasses import dataclass, field
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def snowflake_sdk_available() -> bool:
|
|
11
|
+
try:
|
|
12
|
+
import snowflake.connector # noqa: F401
|
|
13
|
+
except ImportError:
|
|
14
|
+
return False
|
|
15
|
+
return True
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass
|
|
19
|
+
class FakeQueryResult:
|
|
20
|
+
query_id: str
|
|
21
|
+
rows_affected: int = 0
|
|
22
|
+
rows: tuple[dict[str, Any], ...] = ()
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass
|
|
26
|
+
class FakeSnowflakeConnection:
|
|
27
|
+
"""Transactional fake: autocommit is always off.
|
|
28
|
+
|
|
29
|
+
Staged DML lives in a transaction buffer until ``commit()`` or ``rollback()``.
|
|
30
|
+
Every statement receives a stable ``query_id`` for CommitReceipt evidence and
|
|
31
|
+
post-loss reconciliation.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
autocommit: bool = False
|
|
35
|
+
tables: dict[str, list[dict[str, Any]]] = field(default_factory=dict)
|
|
36
|
+
_txn: list[tuple[str, str, list[dict[str, Any]]]] = field(default_factory=list)
|
|
37
|
+
_committed_queries: dict[str, FakeQueryResult] = field(default_factory=dict)
|
|
38
|
+
_pending_queries: dict[str, FakeQueryResult] = field(default_factory=dict)
|
|
39
|
+
_query_seq: int = 0
|
|
40
|
+
|
|
41
|
+
def __post_init__(self) -> None:
|
|
42
|
+
# Enforce transactional path regardless of constructor override.
|
|
43
|
+
self.autocommit = False
|
|
44
|
+
|
|
45
|
+
def execute(
|
|
46
|
+
self,
|
|
47
|
+
sql: str,
|
|
48
|
+
*,
|
|
49
|
+
table: str,
|
|
50
|
+
rows: list[dict[str, Any]] | None = None,
|
|
51
|
+
operation: str = "append",
|
|
52
|
+
) -> FakeQueryResult:
|
|
53
|
+
if self.autocommit:
|
|
54
|
+
raise RuntimeError("autocommit must remain False for transactional path")
|
|
55
|
+
self._query_seq += 1
|
|
56
|
+
query_id = f"sfqid-{self._query_seq:08d}-{uuid.uuid4().hex[:6]}"
|
|
57
|
+
payload = list(rows or [])
|
|
58
|
+
result = FakeQueryResult(query_id=query_id, rows_affected=len(payload), rows=())
|
|
59
|
+
self._txn.append((operation, table, payload))
|
|
60
|
+
self._pending_queries[query_id] = result
|
|
61
|
+
return result
|
|
62
|
+
|
|
63
|
+
def select(self, table: str) -> FakeQueryResult:
|
|
64
|
+
self._query_seq += 1
|
|
65
|
+
query_id = f"sfqid-{self._query_seq:08d}-{uuid.uuid4().hex[:6]}"
|
|
66
|
+
rows = tuple(self.tables.get(table, []))
|
|
67
|
+
result = FakeQueryResult(query_id=query_id, rows_affected=0, rows=rows)
|
|
68
|
+
self._committed_queries[query_id] = result
|
|
69
|
+
return result
|
|
70
|
+
|
|
71
|
+
def commit(self) -> list[str]:
|
|
72
|
+
query_ids: list[str] = []
|
|
73
|
+
for operation, table, rows in self._txn:
|
|
74
|
+
current = list(self.tables.get(table, []))
|
|
75
|
+
if operation in {"overwrite", "replace"}:
|
|
76
|
+
current = list(rows)
|
|
77
|
+
elif operation == "merge":
|
|
78
|
+
# Keyed upsert on "id" when present; else append.
|
|
79
|
+
by_id = {
|
|
80
|
+
r["id"]: dict(r)
|
|
81
|
+
for r in current
|
|
82
|
+
if isinstance(r, dict) and "id" in r
|
|
83
|
+
}
|
|
84
|
+
rest = [r for r in current if not (isinstance(r, dict) and "id" in r)]
|
|
85
|
+
for row in rows:
|
|
86
|
+
if isinstance(row, dict) and "id" in row:
|
|
87
|
+
by_id[row["id"]] = dict(row)
|
|
88
|
+
else:
|
|
89
|
+
rest.append(
|
|
90
|
+
dict(row) if isinstance(row, dict) else {"value": row}
|
|
91
|
+
)
|
|
92
|
+
current = rest + list(by_id.values())
|
|
93
|
+
else:
|
|
94
|
+
current.extend(rows)
|
|
95
|
+
self.tables[table] = current
|
|
96
|
+
query_ids = list(self._pending_queries)
|
|
97
|
+
self._committed_queries.update(self._pending_queries)
|
|
98
|
+
self._pending_queries.clear()
|
|
99
|
+
self._txn.clear()
|
|
100
|
+
return query_ids
|
|
101
|
+
|
|
102
|
+
def rollback(self) -> None:
|
|
103
|
+
self._txn.clear()
|
|
104
|
+
self._pending_queries.clear()
|
|
105
|
+
|
|
106
|
+
def lookup_query(self, query_id: str) -> FakeQueryResult | None:
|
|
107
|
+
return self._committed_queries.get(query_id) or self._pending_queries.get(
|
|
108
|
+
query_id
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
__all__ = [
|
|
113
|
+
"FakeQueryResult",
|
|
114
|
+
"FakeSnowflakeConnection",
|
|
115
|
+
"snowflake_sdk_available",
|
|
116
|
+
]
|