snowflake-sql-api 0.1.2__tar.gz → 0.1.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {snowflake_sql_api-0.1.2 → snowflake_sql_api-0.1.3}/CONTRIBUTING.md +1 -1
- {snowflake_sql_api-0.1.2 → snowflake_sql_api-0.1.3}/PKG-INFO +33 -10
- {snowflake_sql_api-0.1.2 → snowflake_sql_api-0.1.3}/README.md +28 -5
- {snowflake_sql_api-0.1.2 → snowflake_sql_api-0.1.3}/pyproject.toml +17 -19
- {snowflake_sql_api-0.1.2 → snowflake_sql_api-0.1.3}/snowflake_sql_api/_version.py +2 -2
- {snowflake_sql_api-0.1.2 → snowflake_sql_api-0.1.3}/snowflake_sql_api/aclient.py +108 -6
- {snowflake_sql_api-0.1.2 → snowflake_sql_api-0.1.3}/snowflake_sql_api/client.py +96 -6
- snowflake_sql_api-0.1.3/snowflake_sql_api/exports.py +71 -0
- snowflake_sql_api-0.1.3/snowflake_sql_api/pagination.py +131 -0
- {snowflake_sql_api-0.1.2 → snowflake_sql_api-0.1.3}/snowflake_sql_api/types.py +10 -6
- {snowflake_sql_api-0.1.2 → snowflake_sql_api-0.1.3}/tests/conftest.py +6 -0
- {snowflake_sql_api-0.1.2 → snowflake_sql_api-0.1.3}/tests/smoke/test_smoke.py +37 -0
- snowflake_sql_api-0.1.3/tests/streaming_support.py +73 -0
- snowflake_sql_api-0.1.3/tests/test_exports.py +221 -0
- {snowflake_sql_api-0.1.2 → snowflake_sql_api-0.1.3}/tests/test_import_footprint.py +1 -3
- {snowflake_sql_api-0.1.2 → snowflake_sql_api-0.1.3}/tests/test_regression.py +2 -2
- snowflake_sql_api-0.1.3/tests/test_streaming.py +279 -0
- {snowflake_sql_api-0.1.2 → snowflake_sql_api-0.1.3}/tests/test_testing.py +0 -2
- snowflake_sql_api-0.1.2/snowflake_sql_api/pagination.py +0 -53
- {snowflake_sql_api-0.1.2 → snowflake_sql_api-0.1.3}/.gitignore +0 -0
- {snowflake_sql_api-0.1.2 → snowflake_sql_api-0.1.3}/LICENSE +0 -0
- {snowflake_sql_api-0.1.2 → snowflake_sql_api-0.1.3}/snowflake_sql_api/__init__.py +0 -0
- {snowflake_sql_api-0.1.2 → snowflake_sql_api-0.1.3}/snowflake_sql_api/auth.py +0 -0
- {snowflake_sql_api-0.1.2 → snowflake_sql_api-0.1.3}/snowflake_sql_api/bindings.py +0 -0
- {snowflake_sql_api-0.1.2 → snowflake_sql_api-0.1.3}/snowflake_sql_api/cli.py +0 -0
- {snowflake_sql_api-0.1.2 → snowflake_sql_api-0.1.3}/snowflake_sql_api/escaping.py +0 -0
- {snowflake_sql_api-0.1.2 → snowflake_sql_api-0.1.3}/snowflake_sql_api/exceptions.py +0 -0
- {snowflake_sql_api-0.1.2 → snowflake_sql_api-0.1.3}/snowflake_sql_api/py.typed +0 -0
- {snowflake_sql_api-0.1.2 → snowflake_sql_api-0.1.3}/snowflake_sql_api/retry.py +0 -0
- {snowflake_sql_api-0.1.2 → snowflake_sql_api-0.1.3}/snowflake_sql_api/row_mapping.py +0 -0
- {snowflake_sql_api-0.1.2 → snowflake_sql_api-0.1.3}/snowflake_sql_api/testing.py +0 -0
- {snowflake_sql_api-0.1.2 → snowflake_sql_api-0.1.3}/snowflake_sql_api/transport.py +0 -0
- {snowflake_sql_api-0.1.2 → snowflake_sql_api-0.1.3}/tests/__init__.py +0 -0
- {snowflake_sql_api-0.1.2 → snowflake_sql_api-0.1.3}/tests/smoke/__init__.py +0 -0
- {snowflake_sql_api-0.1.2 → snowflake_sql_api-0.1.3}/tests/support.py +0 -0
- {snowflake_sql_api-0.1.2 → snowflake_sql_api-0.1.3}/tests/test_aclient.py +0 -0
- {snowflake_sql_api-0.1.2 → snowflake_sql_api-0.1.3}/tests/test_auth.py +0 -0
- {snowflake_sql_api-0.1.2 → snowflake_sql_api-0.1.3}/tests/test_bindings.py +0 -0
- {snowflake_sql_api-0.1.2 → snowflake_sql_api-0.1.3}/tests/test_cli.py +0 -0
- {snowflake_sql_api-0.1.2 → snowflake_sql_api-0.1.3}/tests/test_client.py +0 -0
- {snowflake_sql_api-0.1.2 → snowflake_sql_api-0.1.3}/tests/test_escaping.py +0 -0
- {snowflake_sql_api-0.1.2 → snowflake_sql_api-0.1.3}/tests/test_package.py +0 -0
- {snowflake_sql_api-0.1.2 → snowflake_sql_api-0.1.3}/tests/test_retry.py +0 -0
- {snowflake_sql_api-0.1.2 → snowflake_sql_api-0.1.3}/tests/test_transport.py +0 -0
- {snowflake_sql_api-0.1.2 → snowflake_sql_api-0.1.3}/tests/test_types.py +0 -0
|
@@ -34,7 +34,7 @@ All four must pass (and `pre-commit run --all-files` is clean). Use `coverage ru
|
|
|
34
34
|
(`snowflake_sql_api.testing`), and `coverage run` starts tracing before that
|
|
35
35
|
plugin is imported so import-time lines are measured correctly. The coverage gate
|
|
36
36
|
is enforced (`fail_under = 89` in `pyproject.toml`); `coverage report` exits
|
|
37
|
-
non-zero below it. CI runs the same checks across Python 3.
|
|
37
|
+
non-zero below it. CI runs the same checks across Python 3.11-3.15.
|
|
38
38
|
|
|
39
39
|
Checklist:
|
|
40
40
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: snowflake-sql-api
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.3
|
|
4
4
|
Summary: Lightweight, pure-Python client for Snowflake's SQL API v2
|
|
5
5
|
Project-URL: Homepage, https://github.com/hampsterx/snowflake-sql-api
|
|
6
6
|
Project-URL: Repository, https://github.com/hampsterx/snowflake-sql-api
|
|
@@ -34,20 +34,20 @@ Classifier: Intended Audience :: Developers
|
|
|
34
34
|
Classifier: License :: OSI Approved :: MIT License
|
|
35
35
|
Classifier: Operating System :: OS Independent
|
|
36
36
|
Classifier: Programming Language :: Python :: 3
|
|
37
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
38
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
39
37
|
Classifier: Programming Language :: Python :: 3.11
|
|
40
38
|
Classifier: Programming Language :: Python :: 3.12
|
|
41
39
|
Classifier: Programming Language :: Python :: 3.13
|
|
40
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
41
|
+
Classifier: Programming Language :: Python :: 3.15
|
|
42
42
|
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
43
43
|
Classifier: Topic :: Database
|
|
44
44
|
Classifier: Typing :: Typed
|
|
45
|
-
Requires-Python: >=3.
|
|
45
|
+
Requires-Python: >=3.11
|
|
46
46
|
Requires-Dist: cryptography>=3.4
|
|
47
47
|
Requires-Dist: httpx>=0.24
|
|
48
48
|
Requires-Dist: pyjwt>=2.4
|
|
49
49
|
Provides-Extra: dev
|
|
50
|
-
Requires-Dist: black<
|
|
50
|
+
Requires-Dist: black<26.6,>=26.5; extra == 'dev'
|
|
51
51
|
Requires-Dist: build>=1.0; extra == 'dev'
|
|
52
52
|
Requires-Dist: coverage[toml]>=7.0; extra == 'dev'
|
|
53
53
|
Requires-Dist: mypy>=1.8; extra == 'dev'
|
|
@@ -64,10 +64,16 @@ Description-Content-Type: text/markdown
|
|
|
64
64
|
|
|
65
65
|
# snowflake-sql-api
|
|
66
66
|
|
|
67
|
+
[](https://github.com/hampsterx/snowflake-sql-api/actions/workflows/ci.yml)
|
|
68
|
+
[](https://pypi.org/project/snowflake-sql-api/)
|
|
69
|
+
[](https://pypi.org/project/snowflake-sql-api/)
|
|
70
|
+
[](LICENSE)
|
|
71
|
+
[](https://peps.python.org/pep-0561/)
|
|
72
|
+
|
|
67
73
|
A lightweight, pure-Python client for [Snowflake's SQL API v2](https://docs.snowflake.com/en/developer-guide/sql-api/index)
|
|
68
74
|
(the `POST /api/v2/statements` REST endpoint).
|
|
69
75
|
|
|
70
|
-
> **Status:
|
|
76
|
+
> **Status: pre-1.0.** The API may still change between minor versions.
|
|
71
77
|
|
|
72
78
|
## Why
|
|
73
79
|
|
|
@@ -87,6 +93,7 @@ stays small and cold-starts quickly. Core dependencies: `httpx`, `PyJWT`,
|
|
|
87
93
|
- Keypair (JWT) authentication, including encrypted private keys
|
|
88
94
|
- Result type coercion (NUMBER/Decimal, dates/timestamps, VARIANT, and more)
|
|
89
95
|
- Multi-partition result handling for large result sets
|
|
96
|
+
- Lazy row streams and bounded row batches, with CSV and JSONL exports
|
|
90
97
|
- Query helpers, DML, batch insert, async statement submission
|
|
91
98
|
- A small CLI for ad-hoc queries
|
|
92
99
|
- Optional pandas (`[pandas]`) and typed-row (`[pydantic]`) helpers, kept out of
|
|
@@ -132,6 +139,22 @@ CLI:
|
|
|
132
139
|
snowflake-sql-api query "SELECT current_version()"
|
|
133
140
|
```
|
|
134
141
|
|
|
142
|
+
Stream large results or export them without collecting every row:
|
|
143
|
+
|
|
144
|
+
```python
|
|
145
|
+
with SnowflakeClient.from_env() as client:
|
|
146
|
+
for batch in client.query_batches("SELECT id, amount FROM sales", batch_size=500):
|
|
147
|
+
process(batch)
|
|
148
|
+
|
|
149
|
+
with open("sales.csv", "w", encoding="utf-8", newline="") as output:
|
|
150
|
+
count = client.export_csv("SELECT id, amount FROM sales", output)
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
Streaming fetches completed results one server partition at a time. Memory scales
|
|
154
|
+
with the largest decoded partition plus the application batch. See
|
|
155
|
+
[streaming results and exports](docs/streaming.md) for row streams, async usage,
|
|
156
|
+
serialization, and partial-output behavior.
|
|
157
|
+
|
|
135
158
|
## Prior art and alternatives
|
|
136
159
|
|
|
137
160
|
`snowflake-sql-api` is not the first REST-based Snowflake client. If it does not
|
|
@@ -146,10 +169,10 @@ fit your needs, one of these might:
|
|
|
146
169
|
| [`rb_snowflake_client`](https://github.com/rinsed-org/rb-snowflake-client) | Ruby | Mature pure-Ruby SQL API client (streaming, connection pooling). |
|
|
147
170
|
| [`rsql`](https://github.com/theseus-rs/rsql) | Rust | Multi-database SQL CLI with a Snowflake SQL API driver. |
|
|
148
171
|
|
|
149
|
-
**How `snowflake-sql-api` differs:** pure Python over `httpx
|
|
150
|
-
the official connector (small install, fast cold start), paired
|
|
151
|
-
clients,
|
|
152
|
-
first-class behavior.
|
|
172
|
+
**How `snowflake-sql-api` differs:** pure Python over `httpx`, with no
|
|
173
|
+
dependency on the official connector (small install, fast cold start), paired
|
|
174
|
+
sync and async clients, keypair auth, type coercion, partition handling, and a
|
|
175
|
+
CLI as tested first-class behavior.
|
|
153
176
|
|
|
154
177
|
Note: the similarly named `snowflake-sql-api-async` and `snowflake-sql-api-client`
|
|
155
178
|
are separate, unaffiliated projects.
|
|
@@ -1,9 +1,15 @@
|
|
|
1
1
|
# snowflake-sql-api
|
|
2
2
|
|
|
3
|
+
[](https://github.com/hampsterx/snowflake-sql-api/actions/workflows/ci.yml)
|
|
4
|
+
[](https://pypi.org/project/snowflake-sql-api/)
|
|
5
|
+
[](https://pypi.org/project/snowflake-sql-api/)
|
|
6
|
+
[](LICENSE)
|
|
7
|
+
[](https://peps.python.org/pep-0561/)
|
|
8
|
+
|
|
3
9
|
A lightweight, pure-Python client for [Snowflake's SQL API v2](https://docs.snowflake.com/en/developer-guide/sql-api/index)
|
|
4
10
|
(the `POST /api/v2/statements` REST endpoint).
|
|
5
11
|
|
|
6
|
-
> **Status:
|
|
12
|
+
> **Status: pre-1.0.** The API may still change between minor versions.
|
|
7
13
|
|
|
8
14
|
## Why
|
|
9
15
|
|
|
@@ -23,6 +29,7 @@ stays small and cold-starts quickly. Core dependencies: `httpx`, `PyJWT`,
|
|
|
23
29
|
- Keypair (JWT) authentication, including encrypted private keys
|
|
24
30
|
- Result type coercion (NUMBER/Decimal, dates/timestamps, VARIANT, and more)
|
|
25
31
|
- Multi-partition result handling for large result sets
|
|
32
|
+
- Lazy row streams and bounded row batches, with CSV and JSONL exports
|
|
26
33
|
- Query helpers, DML, batch insert, async statement submission
|
|
27
34
|
- A small CLI for ad-hoc queries
|
|
28
35
|
- Optional pandas (`[pandas]`) and typed-row (`[pydantic]`) helpers, kept out of
|
|
@@ -68,6 +75,22 @@ CLI:
|
|
|
68
75
|
snowflake-sql-api query "SELECT current_version()"
|
|
69
76
|
```
|
|
70
77
|
|
|
78
|
+
Stream large results or export them without collecting every row:
|
|
79
|
+
|
|
80
|
+
```python
|
|
81
|
+
with SnowflakeClient.from_env() as client:
|
|
82
|
+
for batch in client.query_batches("SELECT id, amount FROM sales", batch_size=500):
|
|
83
|
+
process(batch)
|
|
84
|
+
|
|
85
|
+
with open("sales.csv", "w", encoding="utf-8", newline="") as output:
|
|
86
|
+
count = client.export_csv("SELECT id, amount FROM sales", output)
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
Streaming fetches completed results one server partition at a time. Memory scales
|
|
90
|
+
with the largest decoded partition plus the application batch. See
|
|
91
|
+
[streaming results and exports](docs/streaming.md) for row streams, async usage,
|
|
92
|
+
serialization, and partial-output behavior.
|
|
93
|
+
|
|
71
94
|
## Prior art and alternatives
|
|
72
95
|
|
|
73
96
|
`snowflake-sql-api` is not the first REST-based Snowflake client. If it does not
|
|
@@ -82,10 +105,10 @@ fit your needs, one of these might:
|
|
|
82
105
|
| [`rb_snowflake_client`](https://github.com/rinsed-org/rb-snowflake-client) | Ruby | Mature pure-Ruby SQL API client (streaming, connection pooling). |
|
|
83
106
|
| [`rsql`](https://github.com/theseus-rs/rsql) | Rust | Multi-database SQL CLI with a Snowflake SQL API driver. |
|
|
84
107
|
|
|
85
|
-
**How `snowflake-sql-api` differs:** pure Python over `httpx
|
|
86
|
-
the official connector (small install, fast cold start), paired
|
|
87
|
-
clients,
|
|
88
|
-
first-class behavior.
|
|
108
|
+
**How `snowflake-sql-api` differs:** pure Python over `httpx`, with no
|
|
109
|
+
dependency on the official connector (small install, fast cold start), paired
|
|
110
|
+
sync and async clients, keypair auth, type coercion, partition handling, and a
|
|
111
|
+
CLI as tested first-class behavior.
|
|
89
112
|
|
|
90
113
|
Note: the similarly named `snowflake-sql-api-async` and `snowflake-sql-api-client`
|
|
91
114
|
are separate, unaffiliated projects.
|
|
@@ -7,7 +7,7 @@ name = "snowflake-sql-api"
|
|
|
7
7
|
dynamic = ["version"]
|
|
8
8
|
description = "Lightweight, pure-Python client for Snowflake's SQL API v2"
|
|
9
9
|
readme = "README.md"
|
|
10
|
-
requires-python = ">=3.
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
11
|
license = { file = "LICENSE" }
|
|
12
12
|
authors = [{ name = "hampsterx" }]
|
|
13
13
|
keywords = ["snowflake", "sql", "rest", "api", "httpx", "serverless", "lambda"]
|
|
@@ -17,11 +17,11 @@ classifiers = [
|
|
|
17
17
|
"License :: OSI Approved :: MIT License",
|
|
18
18
|
"Operating System :: OS Independent",
|
|
19
19
|
"Programming Language :: Python :: 3",
|
|
20
|
-
"Programming Language :: Python :: 3.9",
|
|
21
|
-
"Programming Language :: Python :: 3.10",
|
|
22
20
|
"Programming Language :: Python :: 3.11",
|
|
23
21
|
"Programming Language :: Python :: 3.12",
|
|
24
22
|
"Programming Language :: Python :: 3.13",
|
|
23
|
+
"Programming Language :: Python :: 3.14",
|
|
24
|
+
"Programming Language :: Python :: 3.15",
|
|
25
25
|
"Programming Language :: Python :: Implementation :: CPython",
|
|
26
26
|
"Topic :: Database",
|
|
27
27
|
"Typing :: Typed",
|
|
@@ -41,12 +41,10 @@ dev = [
|
|
|
41
41
|
"pytest-asyncio>=0.21",
|
|
42
42
|
"respx>=0.20",
|
|
43
43
|
"ruff>=0.4",
|
|
44
|
-
#
|
|
45
|
-
# pins (rev
|
|
46
|
-
# hand; Dependabot ignores black (.github/dependabot.yml).
|
|
47
|
-
|
|
48
|
-
# a higher cap resolves a different black per interpreter.
|
|
49
|
-
"black>=24.0,<25.10",
|
|
44
|
+
# Held to 26.5.x so CI formats with the same black the pre-commit hook
|
|
45
|
+
# pins (rev 26.5.1 in .pre-commit-config.yaml). Move the two together by
|
|
46
|
+
# hand; Dependabot ignores black (.github/dependabot.yml).
|
|
47
|
+
"black>=26.5,<26.6",
|
|
50
48
|
"mypy>=1.8",
|
|
51
49
|
"build>=1.0",
|
|
52
50
|
"pre-commit>=3.5",
|
|
@@ -89,13 +87,13 @@ include = [
|
|
|
89
87
|
|
|
90
88
|
[tool.ruff]
|
|
91
89
|
line-length = 88
|
|
92
|
-
target-version = "
|
|
90
|
+
target-version = "py311"
|
|
93
91
|
|
|
94
92
|
[tool.ruff.lint]
|
|
95
93
|
select = ["E", "F", "W", "I", "B", "C4", "SIM"]
|
|
96
|
-
# E501 (line length) is owned by black. `UP` (pyupgrade) is
|
|
97
|
-
#
|
|
98
|
-
#
|
|
94
|
+
# E501 (line length) is owned by black. `UP` (pyupgrade) is not enabled yet: the
|
|
95
|
+
# code predates the 3.11 floor and still spells `Optional`/`List`, so turning it
|
|
96
|
+
# on is a ~250-site annotation rewrite of its own.
|
|
99
97
|
ignore = ["E501"]
|
|
100
98
|
|
|
101
99
|
[tool.ruff.lint.per-file-ignores]
|
|
@@ -103,14 +101,14 @@ ignore = ["E501"]
|
|
|
103
101
|
|
|
104
102
|
[tool.black]
|
|
105
103
|
line-length = 88
|
|
106
|
-
# Pin to
|
|
107
|
-
#
|
|
108
|
-
target-version = ["
|
|
104
|
+
# Pin to the floor's grammar: a single floor target keeps black's AST safety
|
|
105
|
+
# check parseable by the interpreter running CI.
|
|
106
|
+
target-version = ["py311"]
|
|
109
107
|
|
|
110
108
|
[tool.mypy]
|
|
111
|
-
#
|
|
112
|
-
#
|
|
113
|
-
python_version = "3.
|
|
109
|
+
# The runtime floor; the pytest matrix (3.11-3.15) imports every module under
|
|
110
|
+
# each supported interpreter.
|
|
111
|
+
python_version = "3.11"
|
|
114
112
|
files = ["snowflake_sql_api"]
|
|
115
113
|
disallow_untyped_defs = true
|
|
116
114
|
disallow_incomplete_defs = true
|
|
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
|
|
|
18
18
|
commit_id: str | None
|
|
19
19
|
__commit_id__: str | None
|
|
20
20
|
|
|
21
|
-
__version__ = version = '0.1.
|
|
22
|
-
__version_tuple__ = version_tuple = (0, 1,
|
|
21
|
+
__version__ = version = '0.1.3'
|
|
22
|
+
__version_tuple__ = version_tuple = (0, 1, 3)
|
|
23
23
|
|
|
24
24
|
__commit_id__ = commit_id = None
|
|
@@ -12,7 +12,17 @@ import os
|
|
|
12
12
|
import time
|
|
13
13
|
import uuid
|
|
14
14
|
from pathlib import Path
|
|
15
|
-
from typing import
|
|
15
|
+
from typing import (
|
|
16
|
+
Any,
|
|
17
|
+
AsyncGenerator,
|
|
18
|
+
Callable,
|
|
19
|
+
Dict,
|
|
20
|
+
List,
|
|
21
|
+
Optional,
|
|
22
|
+
Sequence,
|
|
23
|
+
TextIO,
|
|
24
|
+
Tuple,
|
|
25
|
+
)
|
|
16
26
|
|
|
17
27
|
import httpx
|
|
18
28
|
|
|
@@ -26,7 +36,13 @@ from .exceptions import (
|
|
|
26
36
|
SnowflakeRequestError,
|
|
27
37
|
SnowflakeTimeoutError,
|
|
28
38
|
)
|
|
29
|
-
from .
|
|
39
|
+
from .exports import (
|
|
40
|
+
CSVWriter,
|
|
41
|
+
validate_batch_size,
|
|
42
|
+
validate_export_columns,
|
|
43
|
+
write_jsonl,
|
|
44
|
+
)
|
|
45
|
+
from .pagination import fetch_all_partitions_async, iter_result_rows_async
|
|
30
46
|
from .retry import DEFAULT_RETRY_POLICY, RetryPolicy
|
|
31
47
|
from .transport import AsyncTransport, StatementResponse
|
|
32
48
|
from .types import ColumnMeta, Row, coerce_rows
|
|
@@ -201,13 +217,13 @@ class AsyncSnowflakeClient:
|
|
|
201
217
|
)
|
|
202
218
|
await asyncio.sleep(self._poll_interval)
|
|
203
219
|
|
|
204
|
-
async def
|
|
220
|
+
async def _prepare_result(
|
|
205
221
|
self,
|
|
206
222
|
response: StatementResponse,
|
|
207
223
|
*,
|
|
208
224
|
timeout: Optional[float],
|
|
209
225
|
poll: bool = True,
|
|
210
|
-
) -> Tuple[List[ColumnMeta],
|
|
226
|
+
) -> Tuple[List[ColumnMeta], str, Dict[str, Any]]:
|
|
211
227
|
if response.is_running:
|
|
212
228
|
if not poll:
|
|
213
229
|
raise ResultNotReady(statement_handle=response.statement_handle)
|
|
@@ -220,8 +236,27 @@ class AsyncSnowflakeClient:
|
|
|
220
236
|
handle = response.statement_handle
|
|
221
237
|
if handle is None:
|
|
222
238
|
raise SnowflakeRequestError("completed statement has no handle")
|
|
223
|
-
|
|
224
|
-
|
|
239
|
+
return columns, handle, body
|
|
240
|
+
|
|
241
|
+
async def _collect(
|
|
242
|
+
self,
|
|
243
|
+
response: StatementResponse,
|
|
244
|
+
*,
|
|
245
|
+
timeout: Optional[float],
|
|
246
|
+
poll: bool = True,
|
|
247
|
+
) -> Tuple[List[ColumnMeta], List[List[Any]]]:
|
|
248
|
+
columns, handle, body = await self._prepare_result(
|
|
249
|
+
response, timeout=timeout, poll=poll
|
|
250
|
+
)
|
|
251
|
+
return columns, await fetch_all_partitions_async(self._transport, handle, body)
|
|
252
|
+
|
|
253
|
+
async def _stream_result(
|
|
254
|
+
self, sql: str, params: Params
|
|
255
|
+
) -> Tuple[List[ColumnMeta], AsyncGenerator[Row, None]]:
|
|
256
|
+
columns, handle, body = await self._prepare_result(
|
|
257
|
+
await self._run(sql, params), timeout=self._statement_timeout
|
|
258
|
+
)
|
|
259
|
+
return columns, iter_result_rows_async(self._transport, handle, body, columns)
|
|
225
260
|
|
|
226
261
|
# -- query surface ----------------------------------------------------
|
|
227
262
|
|
|
@@ -233,6 +268,73 @@ class AsyncSnowflakeClient:
|
|
|
233
268
|
)
|
|
234
269
|
return coerce_rows(raw_rows, columns)
|
|
235
270
|
|
|
271
|
+
async def query_stream(
|
|
272
|
+
self, sql: str, params: Params = None
|
|
273
|
+
) -> AsyncGenerator[Row, None]:
|
|
274
|
+
"""Yield converted rows on demand; call directly with async for.
|
|
275
|
+
|
|
276
|
+
Submission starts on first iteration. Keep the client open while
|
|
277
|
+
consuming; await aclose() on a retained stream when stopping early.
|
|
278
|
+
"""
|
|
279
|
+
_, rows = await self._stream_result(sql, params)
|
|
280
|
+
try:
|
|
281
|
+
async for row in rows:
|
|
282
|
+
yield row
|
|
283
|
+
finally:
|
|
284
|
+
await rows.aclose()
|
|
285
|
+
|
|
286
|
+
async def query_batches(
|
|
287
|
+
self, sql: str, params: Params = None, *, batch_size: int = 1000
|
|
288
|
+
) -> AsyncGenerator[List[Row], None]:
|
|
289
|
+
"""Yield batches of at most batch_size rows across partition boundaries."""
|
|
290
|
+
validate_batch_size(batch_size)
|
|
291
|
+
rows = self.query_stream(sql, params)
|
|
292
|
+
try:
|
|
293
|
+
batch: List[Row] = []
|
|
294
|
+
async for row in rows:
|
|
295
|
+
batch.append(row)
|
|
296
|
+
if len(batch) == batch_size:
|
|
297
|
+
yield batch
|
|
298
|
+
batch = []
|
|
299
|
+
if batch:
|
|
300
|
+
yield batch
|
|
301
|
+
finally:
|
|
302
|
+
await rows.aclose()
|
|
303
|
+
|
|
304
|
+
async def export_csv(self, sql: str, output: TextIO, params: Params = None) -> int:
|
|
305
|
+
"""Await query I/O and write CSV to a caller-owned synchronous text sink.
|
|
306
|
+
|
|
307
|
+
Return the data-row count. Failures may leave partial output.
|
|
308
|
+
"""
|
|
309
|
+
columns, rows = await self._stream_result(sql, params)
|
|
310
|
+
try:
|
|
311
|
+
writer = CSVWriter(output, columns)
|
|
312
|
+
count = 0
|
|
313
|
+
async for row in rows:
|
|
314
|
+
writer.write(row)
|
|
315
|
+
count += 1
|
|
316
|
+
finally:
|
|
317
|
+
await rows.aclose()
|
|
318
|
+
return count
|
|
319
|
+
|
|
320
|
+
async def export_jsonl(
|
|
321
|
+
self, sql: str, output: TextIO, params: Params = None
|
|
322
|
+
) -> int:
|
|
323
|
+
"""Await query I/O and write JSONL to a caller-owned synchronous text sink.
|
|
324
|
+
|
|
325
|
+
Return the row count. Failures may leave partial output.
|
|
326
|
+
"""
|
|
327
|
+
columns, rows = await self._stream_result(sql, params)
|
|
328
|
+
try:
|
|
329
|
+
validate_export_columns(columns)
|
|
330
|
+
count = 0
|
|
331
|
+
async for row in rows:
|
|
332
|
+
write_jsonl(output, row)
|
|
333
|
+
count += 1
|
|
334
|
+
finally:
|
|
335
|
+
await rows.aclose()
|
|
336
|
+
return count
|
|
337
|
+
|
|
236
338
|
async def query_one(self, sql: str, params: Params = None) -> Optional[Row]:
|
|
237
339
|
"""Return the first row, or ``None`` if the result is empty."""
|
|
238
340
|
rows = await self.query(sql, params)
|
|
@@ -11,8 +11,19 @@ from __future__ import annotations
|
|
|
11
11
|
import os
|
|
12
12
|
import time
|
|
13
13
|
import uuid
|
|
14
|
+
from contextlib import closing
|
|
14
15
|
from pathlib import Path
|
|
15
|
-
from typing import
|
|
16
|
+
from typing import (
|
|
17
|
+
Any,
|
|
18
|
+
Callable,
|
|
19
|
+
Dict,
|
|
20
|
+
Generator,
|
|
21
|
+
List,
|
|
22
|
+
Optional,
|
|
23
|
+
Sequence,
|
|
24
|
+
TextIO,
|
|
25
|
+
Tuple,
|
|
26
|
+
)
|
|
16
27
|
|
|
17
28
|
import httpx
|
|
18
29
|
|
|
@@ -25,7 +36,13 @@ from .exceptions import (
|
|
|
25
36
|
SnowflakeRequestError,
|
|
26
37
|
SnowflakeTimeoutError,
|
|
27
38
|
)
|
|
28
|
-
from .
|
|
39
|
+
from .exports import (
|
|
40
|
+
CSVWriter,
|
|
41
|
+
validate_batch_size,
|
|
42
|
+
validate_export_columns,
|
|
43
|
+
write_jsonl,
|
|
44
|
+
)
|
|
45
|
+
from .pagination import fetch_all_partitions, iter_result_rows
|
|
29
46
|
from .retry import DEFAULT_RETRY_POLICY, RetryPolicy
|
|
30
47
|
from .transport import StatementResponse, Transport
|
|
31
48
|
from .types import ColumnMeta, Row, coerce_rows
|
|
@@ -246,13 +263,13 @@ class SnowflakeClient:
|
|
|
246
263
|
)
|
|
247
264
|
time.sleep(self._poll_interval)
|
|
248
265
|
|
|
249
|
-
def
|
|
266
|
+
def _prepare_result(
|
|
250
267
|
self,
|
|
251
268
|
response: StatementResponse,
|
|
252
269
|
*,
|
|
253
270
|
timeout: Optional[float],
|
|
254
271
|
poll: bool = True,
|
|
255
|
-
) -> Tuple[List[ColumnMeta],
|
|
272
|
+
) -> Tuple[List[ColumnMeta], str, Dict[str, Any]]:
|
|
256
273
|
if response.is_running:
|
|
257
274
|
if not poll:
|
|
258
275
|
raise ResultNotReady(statement_handle=response.statement_handle)
|
|
@@ -265,8 +282,27 @@ class SnowflakeClient:
|
|
|
265
282
|
handle = response.statement_handle
|
|
266
283
|
if handle is None:
|
|
267
284
|
raise SnowflakeRequestError("completed statement has no handle")
|
|
268
|
-
|
|
269
|
-
|
|
285
|
+
return columns, handle, body
|
|
286
|
+
|
|
287
|
+
def _collect(
|
|
288
|
+
self,
|
|
289
|
+
response: StatementResponse,
|
|
290
|
+
*,
|
|
291
|
+
timeout: Optional[float],
|
|
292
|
+
poll: bool = True,
|
|
293
|
+
) -> Tuple[List[ColumnMeta], List[List[Any]]]:
|
|
294
|
+
columns, handle, body = self._prepare_result(
|
|
295
|
+
response, timeout=timeout, poll=poll
|
|
296
|
+
)
|
|
297
|
+
return columns, fetch_all_partitions(self._transport, handle, body)
|
|
298
|
+
|
|
299
|
+
def _stream_result(
|
|
300
|
+
self, sql: str, params: Params
|
|
301
|
+
) -> Tuple[List[ColumnMeta], Generator[Row, None, None]]:
|
|
302
|
+
columns, handle, body = self._prepare_result(
|
|
303
|
+
self._run(sql, params), timeout=self._statement_timeout
|
|
304
|
+
)
|
|
305
|
+
return columns, iter_result_rows(self._transport, handle, body, columns)
|
|
270
306
|
|
|
271
307
|
# -- query surface ----------------------------------------------------
|
|
272
308
|
|
|
@@ -276,6 +312,60 @@ class SnowflakeClient:
|
|
|
276
312
|
columns, raw_rows = self._collect(response, timeout=self._statement_timeout)
|
|
277
313
|
return coerce_rows(raw_rows, columns)
|
|
278
314
|
|
|
315
|
+
def query_stream(
|
|
316
|
+
self, sql: str, params: Params = None
|
|
317
|
+
) -> Generator[Row, None, None]:
|
|
318
|
+
"""Yield converted rows, submitting on first iteration without prefetch.
|
|
319
|
+
|
|
320
|
+
Memory scales with the largest decoded server partition. Keep the client
|
|
321
|
+
open while iterating; close a retained stream when stopping early.
|
|
322
|
+
"""
|
|
323
|
+
_, rows = self._stream_result(sql, params)
|
|
324
|
+
yield from rows
|
|
325
|
+
|
|
326
|
+
def query_batches(
|
|
327
|
+
self, sql: str, params: Params = None, *, batch_size: int = 1000
|
|
328
|
+
) -> Generator[List[Row], None, None]:
|
|
329
|
+
"""Yield batches of at most batch_size rows across partition boundaries."""
|
|
330
|
+
validate_batch_size(batch_size)
|
|
331
|
+
with closing(self.query_stream(sql, params)) as rows:
|
|
332
|
+
batch: List[Row] = []
|
|
333
|
+
for row in rows:
|
|
334
|
+
batch.append(row)
|
|
335
|
+
if len(batch) == batch_size:
|
|
336
|
+
yield batch
|
|
337
|
+
batch = []
|
|
338
|
+
if batch:
|
|
339
|
+
yield batch
|
|
340
|
+
|
|
341
|
+
def export_csv(self, sql: str, output: TextIO, params: Params = None) -> int:
|
|
342
|
+
"""Stream CSV with a header to an open text sink; return the data-row count.
|
|
343
|
+
|
|
344
|
+
The caller owns the sink. Failures propagate and may leave partial output.
|
|
345
|
+
"""
|
|
346
|
+
columns, rows = self._stream_result(sql, params)
|
|
347
|
+
with closing(rows):
|
|
348
|
+
writer = CSVWriter(output, columns)
|
|
349
|
+
count = 0
|
|
350
|
+
for row in rows:
|
|
351
|
+
writer.write(row)
|
|
352
|
+
count += 1
|
|
353
|
+
return count
|
|
354
|
+
|
|
355
|
+
def export_jsonl(self, sql: str, output: TextIO, params: Params = None) -> int:
|
|
356
|
+
"""Stream JSON objects to an open text sink; return the row count.
|
|
357
|
+
|
|
358
|
+
The caller owns the sink. Failures propagate and may leave partial output.
|
|
359
|
+
"""
|
|
360
|
+
columns, rows = self._stream_result(sql, params)
|
|
361
|
+
with closing(rows):
|
|
362
|
+
validate_export_columns(columns)
|
|
363
|
+
count = 0
|
|
364
|
+
for row in rows:
|
|
365
|
+
write_jsonl(output, row)
|
|
366
|
+
count += 1
|
|
367
|
+
return count
|
|
368
|
+
|
|
279
369
|
def query_one(self, sql: str, params: Params = None) -> Optional[Row]:
|
|
280
370
|
"""Return the first row, or ``None`` if the result is empty."""
|
|
281
371
|
rows = self.query(sql, params)
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"""CSV and JSONL serialization for caller-owned text streams."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import csv
|
|
6
|
+
import json
|
|
7
|
+
from datetime import date, datetime, time
|
|
8
|
+
from decimal import Decimal
|
|
9
|
+
from typing import Any, List, TextIO
|
|
10
|
+
|
|
11
|
+
from .exceptions import SnowflakeConfigError
|
|
12
|
+
from .types import ColumnMeta, Row
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def validate_batch_size(batch_size: int) -> None:
|
|
16
|
+
if (
|
|
17
|
+
isinstance(batch_size, bool)
|
|
18
|
+
or not isinstance(batch_size, int)
|
|
19
|
+
or batch_size <= 0
|
|
20
|
+
):
|
|
21
|
+
raise SnowflakeConfigError("batch_size must be a positive integer")
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def validate_export_columns(columns: List[ColumnMeta]) -> None:
|
|
25
|
+
names = [column.name for column in columns]
|
|
26
|
+
if len(names) != len(set(names)):
|
|
27
|
+
raise SnowflakeConfigError(
|
|
28
|
+
"exports require unique column names; use distinct SQL aliases"
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _json_default(value: Any) -> str:
|
|
33
|
+
if isinstance(value, Decimal):
|
|
34
|
+
return str(value)
|
|
35
|
+
if isinstance(value, (datetime, date, time)):
|
|
36
|
+
return value.isoformat()
|
|
37
|
+
if isinstance(value, (bytes, bytearray)):
|
|
38
|
+
return bytes(value).hex()
|
|
39
|
+
raise TypeError(f"cannot serialize {type(value).__name__}")
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _json(value: Any) -> str:
|
|
43
|
+
return json.dumps(value, default=_json_default, ensure_ascii=False, allow_nan=False)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _csv_value(value: Any) -> Any:
|
|
47
|
+
if value is None:
|
|
48
|
+
return ""
|
|
49
|
+
if isinstance(value, (dict, list, tuple, bool)):
|
|
50
|
+
return _json(value)
|
|
51
|
+
if isinstance(value, (Decimal, datetime, date, time, bytes, bytearray)):
|
|
52
|
+
return _json_default(value)
|
|
53
|
+
return value
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class CSVWriter:
|
|
57
|
+
"""Write a metadata-derived header, then one converted row at a time."""
|
|
58
|
+
|
|
59
|
+
def __init__(self, output: TextIO, columns: List[ColumnMeta]) -> None:
|
|
60
|
+
validate_export_columns(columns)
|
|
61
|
+
self._names = [column.name for column in columns]
|
|
62
|
+
self._writer = csv.writer(output)
|
|
63
|
+
self._writer.writerow(self._names)
|
|
64
|
+
|
|
65
|
+
def write(self, row: Row) -> None:
|
|
66
|
+
self._writer.writerow([_csv_value(row[name]) for name in self._names])
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def write_jsonl(output: TextIO, row: Row) -> None:
|
|
70
|
+
"""Write exactly one JSON record, using the CLI's native-value encodings."""
|
|
71
|
+
output.write(_json(row) + "\n")
|