fakesnow 0.7.1__tar.gz → 0.8.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fakesnow-0.7.1/fakesnow.egg-info → fakesnow-0.8.1}/PKG-INFO +24 -3
- {fakesnow-0.7.1 → fakesnow-0.8.1}/README.md +22 -1
- fakesnow-0.8.1/fakesnow/__main__.py +4 -0
- fakesnow-0.8.1/fakesnow/cli.py +28 -0
- {fakesnow-0.7.1 → fakesnow-0.8.1}/fakesnow/fakes.py +79 -16
- {fakesnow-0.7.1 → fakesnow-0.8.1}/fakesnow/info_schema.py +3 -4
- {fakesnow-0.7.1 → fakesnow-0.8.1}/fakesnow/transforms.py +74 -14
- {fakesnow-0.7.1 → fakesnow-0.8.1/fakesnow.egg-info}/PKG-INFO +24 -3
- {fakesnow-0.7.1 → fakesnow-0.8.1}/fakesnow.egg-info/SOURCES.txt +4 -0
- fakesnow-0.8.1/fakesnow.egg-info/entry_points.txt +2 -0
- {fakesnow-0.7.1 → fakesnow-0.8.1}/fakesnow.egg-info/requires.txt +1 -1
- {fakesnow-0.7.1 → fakesnow-0.8.1}/fakesnow.egg-info/top_level.txt +1 -0
- {fakesnow-0.7.1 → fakesnow-0.8.1}/pyproject.toml +7 -2
- fakesnow-0.8.1/tests/test_cli.py +17 -0
- {fakesnow-0.7.1 → fakesnow-0.8.1}/tests/test_fakes.py +170 -52
- {fakesnow-0.7.1 → fakesnow-0.8.1}/tests/test_patch.py +1 -1
- {fakesnow-0.7.1 → fakesnow-0.8.1}/tests/test_transforms.py +36 -3
- {fakesnow-0.7.1 → fakesnow-0.8.1}/LICENSE +0 -0
- {fakesnow-0.7.1 → fakesnow-0.8.1}/MANIFEST.in +0 -0
- {fakesnow-0.7.1 → fakesnow-0.8.1}/fakesnow/__init__.py +0 -0
- {fakesnow-0.7.1 → fakesnow-0.8.1}/fakesnow/checks.py +0 -0
- {fakesnow-0.7.1 → fakesnow-0.8.1}/fakesnow/expr.py +0 -0
- {fakesnow-0.7.1 → fakesnow-0.8.1}/fakesnow/fixtures.py +0 -0
- {fakesnow-0.7.1 → fakesnow-0.8.1}/fakesnow/py.typed +0 -0
- {fakesnow-0.7.1 → fakesnow-0.8.1}/fakesnow.egg-info/dependency_links.txt +0 -0
- {fakesnow-0.7.1 → fakesnow-0.8.1}/setup.cfg +0 -0
- {fakesnow-0.7.1 → fakesnow-0.8.1}/tests/test_checks.py +0 -0
- {fakesnow-0.7.1 → fakesnow-0.8.1}/tests/test_expr.py +0 -0
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: fakesnow
|
3
|
-
Version: 0.
|
3
|
+
Version: 0.8.1
|
4
4
|
Summary: Fake Snowflake Connector for Python. Run Snowflake DB locally.
|
5
5
|
License: MIT License
|
6
6
|
|
@@ -32,7 +32,7 @@ License-File: LICENSE
|
|
32
32
|
Requires-Dist: duckdb~=0.9.2
|
33
33
|
Requires-Dist: pyarrow
|
34
34
|
Requires-Dist: snowflake-connector-python
|
35
|
-
Requires-Dist: sqlglot~=
|
35
|
+
Requires-Dist: sqlglot~=20.5.0
|
36
36
|
Provides-Extra: dev
|
37
37
|
Requires-Dist: black~=23.9; extra == "dev"
|
38
38
|
Requires-Dist: build~=1.0; extra == "dev"
|
@@ -63,6 +63,24 @@ pip install fakesnow
|
|
63
63
|
|
64
64
|
## Usage
|
65
65
|
|
66
|
+
Run script.py with fakesnow:
|
67
|
+
|
68
|
+
```shell
|
69
|
+
fakesnow script.py
|
70
|
+
```
|
71
|
+
|
72
|
+
Or pytest
|
73
|
+
|
74
|
+
```shell
|
75
|
+
fakesnow -m pytest
|
76
|
+
```
|
77
|
+
|
78
|
+
`fakesnow` executes `fakesnow.patch` before running the script or module.
|
79
|
+
|
80
|
+
### fakesnow.patch
|
81
|
+
|
82
|
+
eg:
|
83
|
+
|
66
84
|
```python
|
67
85
|
import fakesnow
|
68
86
|
import snowflake.connector
|
@@ -91,6 +109,8 @@ with fakesnow.patch("mymodule.write_pandas"):
|
|
91
109
|
...
|
92
110
|
```
|
93
111
|
|
112
|
+
### pytest fixtures
|
113
|
+
|
94
114
|
pytest [fixtures](fakesnow/fixtures.py) are provided for testing. Example _conftest.py_:
|
95
115
|
|
96
116
|
```python
|
@@ -146,7 +166,8 @@ For more detail see [tests/test_fakes.py](tests/test_fakes.py)
|
|
146
166
|
|
147
167
|
## Caveats
|
148
168
|
|
149
|
-
-
|
169
|
+
- The order of rows is non deterministic and may not match Snowflake unless ORDER BY is fully specified.
|
170
|
+
- VARCHAR field sizes are not enforced. Unlike Snowflake which errors with "User character length limit (xxx) exceeded by string" when an inserted string the exceeds the column limit.
|
150
171
|
|
151
172
|
## Contributing
|
152
173
|
|
@@ -14,6 +14,24 @@ pip install fakesnow
|
|
14
14
|
|
15
15
|
## Usage
|
16
16
|
|
17
|
+
Run script.py with fakesnow:
|
18
|
+
|
19
|
+
```shell
|
20
|
+
fakesnow script.py
|
21
|
+
```
|
22
|
+
|
23
|
+
Or pytest
|
24
|
+
|
25
|
+
```shell
|
26
|
+
fakesnow -m pytest
|
27
|
+
```
|
28
|
+
|
29
|
+
`fakesnow` executes `fakesnow.patch` before running the script or module.
|
30
|
+
|
31
|
+
### fakesnow.patch
|
32
|
+
|
33
|
+
eg:
|
34
|
+
|
17
35
|
```python
|
18
36
|
import fakesnow
|
19
37
|
import snowflake.connector
|
@@ -42,6 +60,8 @@ with fakesnow.patch("mymodule.write_pandas"):
|
|
42
60
|
...
|
43
61
|
```
|
44
62
|
|
63
|
+
### pytest fixtures
|
64
|
+
|
45
65
|
pytest [fixtures](fakesnow/fixtures.py) are provided for testing. Example _conftest.py_:
|
46
66
|
|
47
67
|
```python
|
@@ -97,7 +117,8 @@ For more detail see [tests/test_fakes.py](tests/test_fakes.py)
|
|
97
117
|
|
98
118
|
## Caveats
|
99
119
|
|
100
|
-
-
|
120
|
+
- The order of rows is non deterministic and may not match Snowflake unless ORDER BY is fully specified.
|
121
|
+
- VARCHAR field sizes are not enforced. Unlike Snowflake which errors with "User character length limit (xxx) exceeded by string" when an inserted string the exceeds the column limit.
|
101
122
|
|
102
123
|
## Contributing
|
103
124
|
|
@@ -0,0 +1,28 @@
|
|
1
|
+
import runpy
|
2
|
+
import sys
|
3
|
+
from collections.abc import Sequence
|
4
|
+
|
5
|
+
import fakesnow
|
6
|
+
|
7
|
+
USAGE = "Usage: fakesnow <path> | -m <module> [<arg>]..."
|
8
|
+
|
9
|
+
|
10
|
+
def main(args: Sequence[str] = sys.argv) -> int:
|
11
|
+
if len(args) < 2 or (len(args) == 2 and args[1] == "-m"):
|
12
|
+
print(USAGE, file=sys.stderr)
|
13
|
+
return 42
|
14
|
+
|
15
|
+
with fakesnow.patch():
|
16
|
+
if args[1] == "-m":
|
17
|
+
module = args[2]
|
18
|
+
sys.argv = args[2:]
|
19
|
+
|
20
|
+
# add current directory to path to mimic python -m
|
21
|
+
sys.path.insert(0, "")
|
22
|
+
runpy.run_module(module, run_name="__main__", alter_sys=True)
|
23
|
+
else:
|
24
|
+
path = args[1]
|
25
|
+
sys.argv = args[1:]
|
26
|
+
runpy.run_path(path, run_name="__main__")
|
27
|
+
|
28
|
+
return 0
|
@@ -1,9 +1,12 @@
|
|
1
1
|
from __future__ import annotations
|
2
2
|
|
3
|
+
import os
|
3
4
|
import re
|
5
|
+
import sys
|
4
6
|
from collections.abc import Iterable, Iterator, Sequence
|
7
|
+
from string import Template
|
5
8
|
from types import TracebackType
|
6
|
-
from typing import TYPE_CHECKING, Any, Literal, Optional,
|
9
|
+
from typing import TYPE_CHECKING, Any, Literal, Optional, cast
|
7
10
|
|
8
11
|
import duckdb
|
9
12
|
|
@@ -26,6 +29,12 @@ import fakesnow.info_schema as info_schema
|
|
26
29
|
import fakesnow.transforms as transforms
|
27
30
|
|
28
31
|
SCHEMA_UNSET = "schema_unset"
|
32
|
+
SUCCESS_SQL = "SELECT 'Statement executed successfully.' as status"
|
33
|
+
DATABASE_CREATED_SQL = Template("SELECT 'Database ${name} successfully created.' as status")
|
34
|
+
TABLE_CREATED_SQL = Template("SELECT 'Table ${name} successfully created.' as status")
|
35
|
+
DROPPED_SQL = Template("SELECT '${name} successfully dropped.' as status")
|
36
|
+
SCHEMA_CREATED_SQL = Template("SELECT 'Schema ${name} successfully created.' as status")
|
37
|
+
INSERTED_SQL = Template("SELECT ${count} as 'number of rows inserted'")
|
29
38
|
|
30
39
|
|
31
40
|
class FakeSnowflakeCursor:
|
@@ -49,6 +58,7 @@ class FakeSnowflakeCursor:
|
|
49
58
|
self._last_sql = None
|
50
59
|
self._last_params = None
|
51
60
|
self._sqlstate = None
|
61
|
+
self._arraysize = 1
|
52
62
|
self._converter = snowflake.connector.converter.SnowflakeConverter()
|
53
63
|
|
54
64
|
def __enter__(self) -> Self:
|
@@ -62,6 +72,19 @@ class FakeSnowflakeCursor:
|
|
62
72
|
) -> bool:
|
63
73
|
return False
|
64
74
|
|
75
|
+
@property
|
76
|
+
def arraysize(self) -> int:
|
77
|
+
return self._arraysize
|
78
|
+
|
79
|
+
@arraysize.setter
|
80
|
+
def arraysize(self, value: int) -> None:
|
81
|
+
self._arraysize = value
|
82
|
+
|
83
|
+
def close(self) -> bool:
|
84
|
+
self._last_sql = None
|
85
|
+
self._last_params = None
|
86
|
+
return True
|
87
|
+
|
65
88
|
def describe(self, command: str, *args: Any, **kwargs: Any) -> list[ResultMetadata]:
|
66
89
|
"""Return the schema of the result without executing the query.
|
67
90
|
|
@@ -79,8 +102,9 @@ class FakeSnowflakeCursor:
|
|
79
102
|
def description(self) -> list[ResultMetadata]:
|
80
103
|
# use a cursor to avoid destroying an unfetched result on the main connection
|
81
104
|
with self._duck_conn.cursor() as cur:
|
82
|
-
|
83
|
-
assert self._conn.
|
105
|
+
# TODO: allow sql alchemy connection with no database or schema
|
106
|
+
assert self._conn.database, ".description not implemented when database is None"
|
107
|
+
assert self._conn.schema, ".description not implemented when schema is None"
|
84
108
|
|
85
109
|
# match database and schema used on the main connection
|
86
110
|
cur.execute(f"SET SCHEMA = '{self._conn.database}.{self._conn.schema}'")
|
@@ -145,6 +169,8 @@ class FakeSnowflakeCursor:
|
|
145
169
|
.transform(transforms.parse_json)
|
146
170
|
# indices_to_json_extract must be before regex_substr
|
147
171
|
.transform(transforms.indices_to_json_extract)
|
172
|
+
.transform(transforms.json_extract_as_varchar)
|
173
|
+
.transform(transforms.flatten)
|
148
174
|
.transform(transforms.regex_replace)
|
149
175
|
.transform(transforms.regex_substr)
|
150
176
|
.transform(transforms.values_columns)
|
@@ -162,7 +188,8 @@ class FakeSnowflakeCursor:
|
|
162
188
|
try:
|
163
189
|
self._last_sql = sql
|
164
190
|
self._last_params = params
|
165
|
-
|
191
|
+
if os.environ.get("FAKESNOW_DEBUG"):
|
192
|
+
print(f"{sql};", file=sys.stderr)
|
166
193
|
self._duck_conn.execute(sql, params)
|
167
194
|
except duckdb.BinderException as e:
|
168
195
|
msg = e.args[0]
|
@@ -171,6 +198,15 @@ class FakeSnowflakeCursor:
|
|
171
198
|
# minimal processing to make it look like a snowflake exception, message content may differ
|
172
199
|
msg = cast(str, e.args[0]).split("\n")[0]
|
173
200
|
raise snowflake.connector.errors.ProgrammingError(msg=msg, errno=2003, sqlstate="42S02") from None
|
201
|
+
except duckdb.TransactionException as e:
|
202
|
+
if "cannot rollback - no transaction is active" in str(
|
203
|
+
e
|
204
|
+
) or "cannot commit - no transaction is active" in str(e):
|
205
|
+
# snowflake doesn't error on rollback or commit outside a tx
|
206
|
+
self._duck_conn.execute(SUCCESS_SQL)
|
207
|
+
self._last_sql = SUCCESS_SQL
|
208
|
+
else:
|
209
|
+
raise e
|
174
210
|
|
175
211
|
if cmd == "USE DATABASE" and (ident := expression.find(exp.Identifier)) and isinstance(ident.this, str):
|
176
212
|
self._conn.database = ident.this.upper()
|
@@ -183,6 +219,33 @@ class FakeSnowflakeCursor:
|
|
183
219
|
if create_db_name := transformed.args.get("create_db_name"):
|
184
220
|
# we created a new database, so create the info schema extensions
|
185
221
|
self._duck_conn.execute(info_schema.creation_sql(create_db_name))
|
222
|
+
created_sql = DATABASE_CREATED_SQL.substitute(name=create_db_name)
|
223
|
+
self._duck_conn.execute(created_sql)
|
224
|
+
self._last_sql = created_sql
|
225
|
+
|
226
|
+
if cmd == "CREATE SCHEMA" and (ident := expression.find(exp.Identifier)) and isinstance(ident.this, str):
|
227
|
+
name = ident.this if ident.quoted else ident.this.upper()
|
228
|
+
created_sql = SCHEMA_CREATED_SQL.substitute(name=name)
|
229
|
+
self._duck_conn.execute(created_sql)
|
230
|
+
self._last_sql = created_sql
|
231
|
+
|
232
|
+
if cmd == "CREATE TABLE" and (ident := expression.find(exp.Identifier)) and isinstance(ident.this, str):
|
233
|
+
name = ident.this if ident.quoted else ident.this.upper()
|
234
|
+
created_sql = TABLE_CREATED_SQL.substitute(name=name)
|
235
|
+
self._duck_conn.execute(created_sql)
|
236
|
+
self._last_sql = created_sql
|
237
|
+
|
238
|
+
if cmd.startswith("DROP") and (ident := expression.find(exp.Identifier)) and isinstance(ident.this, str):
|
239
|
+
name = ident.this if ident.quoted else ident.this.upper()
|
240
|
+
dropped_sql = DROPPED_SQL.substitute(name=name)
|
241
|
+
self._duck_conn.execute(dropped_sql)
|
242
|
+
self._last_sql = dropped_sql
|
243
|
+
|
244
|
+
if cmd == "INSERT":
|
245
|
+
(count,) = self._duck_conn.fetchall()[0]
|
246
|
+
inserted_sql = INSERTED_SQL.substitute(count=count)
|
247
|
+
self._duck_conn.execute(inserted_sql)
|
248
|
+
self._last_sql = inserted_sql
|
186
249
|
|
187
250
|
if table_comment := cast(tuple[exp.Table, str], transformed.args.get("table_comment")):
|
188
251
|
# record table comment
|
@@ -232,19 +295,22 @@ class FakeSnowflakeCursor:
|
|
232
295
|
return self._duck_conn.fetch_df()
|
233
296
|
|
234
297
|
def fetchone(self) -> dict | tuple | None:
|
298
|
+
result = self.fetchmany(1)
|
299
|
+
return result[0] if result else None
|
300
|
+
|
301
|
+
def fetchmany(self, size: int | None = None) -> list[tuple] | list[dict]:
|
302
|
+
# https://peps.python.org/pep-0249/#fetchmany
|
303
|
+
size = size or self._arraysize
|
235
304
|
if not self._use_dict_result:
|
236
|
-
return cast(
|
305
|
+
return cast(list[tuple], self._duck_conn.fetchmany(size))
|
237
306
|
|
238
307
|
if not self._arrow_table:
|
239
308
|
self._arrow_table = self._duck_conn.fetch_arrow_table()
|
240
|
-
self.
|
309
|
+
self._arrow_table_fetch_index = -size
|
241
310
|
|
242
|
-
self.
|
311
|
+
self._arrow_table_fetch_index += size
|
243
312
|
|
244
|
-
|
245
|
-
return self._arrow_table.take([self._arrow_table_fetch_one_index]).to_pylist()[0]
|
246
|
-
except pyarrow.lib.ArrowIndexError:
|
247
|
-
return None
|
313
|
+
return self._arrow_table.slice(offset=self._arrow_table_fetch_index, length=size).to_pylist()
|
248
314
|
|
249
315
|
def get_result_batches(self) -> list[ResultBatch] | None:
|
250
316
|
# rows_per_batch is approximate
|
@@ -322,13 +388,10 @@ class FakeSnowflakeCursor:
|
|
322
388
|
return ResultMetadata(
|
323
389
|
name=column_name, type_code=12, display_size=None, internal_size=None, precision=0, scale=9, is_nullable=True # noqa: E501
|
324
390
|
)
|
325
|
-
elif column_type == "JSON[]":
|
326
|
-
return ResultMetadata(
|
327
|
-
name=column_name, type_code=10, display_size=None, internal_size=None, precision=None, scale=None, is_nullable=True # noqa: E501
|
328
|
-
)
|
329
391
|
elif column_type == "JSON":
|
392
|
+
# TODO: correctly map OBJECT and ARRAY see https://github.com/tekumara/fakesnow/issues/26
|
330
393
|
return ResultMetadata(
|
331
|
-
name=column_name, type_code=
|
394
|
+
name=column_name, type_code=5, display_size=None, internal_size=None, precision=None, scale=None, is_nullable=True # noqa: E501
|
332
395
|
)
|
333
396
|
else:
|
334
397
|
# TODO handle more types
|
@@ -1,7 +1,7 @@
|
|
1
|
-
from string import Template
|
2
|
-
|
3
1
|
"""Info schema extension tables/views used for storing snowflake metadata not captured by duckdb."""
|
4
2
|
|
3
|
+
from string import Template
|
4
|
+
|
5
5
|
# use ext prefix in columns to disambiguate when joining with information_schema.tables
|
6
6
|
SQL_CREATE_INFORMATION_SCHEMA_TABLES_EXT = Template(
|
7
7
|
"""
|
@@ -40,8 +40,7 @@ case when starts_with(data_type, 'DECIMAL') or data_type='BIGINT' then 'NUMBER'
|
|
40
40
|
when data_type='DOUBLE' then 'FLOAT'
|
41
41
|
when data_type='BLOB' then 'BINARY'
|
42
42
|
when data_type='TIMESTAMP' then 'TIMESTAMP_NTZ'
|
43
|
-
when data_type='JSON
|
44
|
-
when data_type='JSON' then 'OBJECT'
|
43
|
+
when data_type='JSON' then 'VARIANT'
|
45
44
|
else data_type end as data_type,
|
46
45
|
ext_character_maximum_length as character_maximum_length, ext_character_octet_length as character_octet_length,
|
47
46
|
case when data_type='BIGINT' then 38
|
@@ -155,6 +155,49 @@ def extract_text_length(expression: exp.Expression) -> exp.Expression:
|
|
155
155
|
return expression
|
156
156
|
|
157
157
|
|
158
|
+
def flatten(expression: exp.Expression) -> exp.Expression:
|
159
|
+
"""Flatten an array.
|
160
|
+
|
161
|
+
See https://docs.snowflake.com/en/sql-reference/functions/flatten
|
162
|
+
|
163
|
+
TODO: return index.
|
164
|
+
TODO: support objects.
|
165
|
+
"""
|
166
|
+
if (
|
167
|
+
isinstance(expression, exp.Lateral)
|
168
|
+
and isinstance(expression.this, exp.Explode)
|
169
|
+
and (alias := expression.args.get("alias"))
|
170
|
+
# always true; when no explicit alias provided this will be _flattened
|
171
|
+
and isinstance(alias, exp.TableAlias)
|
172
|
+
):
|
173
|
+
explode_expression = expression.this.this.expression
|
174
|
+
|
175
|
+
return exp.Lateral(
|
176
|
+
this=exp.Unnest(
|
177
|
+
expressions=[
|
178
|
+
exp.Anonymous(
|
179
|
+
# duckdb unnests in reserve, so we reverse the list to match
|
180
|
+
# the order of the original array (and snowflake)
|
181
|
+
this="list_reverse",
|
182
|
+
expressions=[
|
183
|
+
exp.Cast(
|
184
|
+
this=explode_expression,
|
185
|
+
to=exp.DataType(
|
186
|
+
this=exp.DataType.Type.ARRAY,
|
187
|
+
expressions=[exp.DataType(this=exp.DataType.Type.JSON, nested=False, prefix=False)],
|
188
|
+
nested=True,
|
189
|
+
),
|
190
|
+
)
|
191
|
+
],
|
192
|
+
)
|
193
|
+
]
|
194
|
+
),
|
195
|
+
alias=exp.TableAlias(this=alias.this, columns=[exp.Identifier(this="VALUE", quoted=False)]),
|
196
|
+
)
|
197
|
+
|
198
|
+
return expression
|
199
|
+
|
200
|
+
|
158
201
|
def float_to_double(expression: exp.Expression) -> exp.Expression:
|
159
202
|
"""Convert float to double for 64 bit precision.
|
160
203
|
|
@@ -176,10 +219,10 @@ def indices_to_json_extract(expression: exp.Expression) -> exp.Expression:
|
|
176
219
|
and object indices, see
|
177
220
|
https://docs.snowflake.com/en/sql-reference/data-types-semistructured#accessing-elements-of-an-object-by-key
|
178
221
|
|
179
|
-
Duckdb uses the -> operator,
|
222
|
+
Duckdb uses the -> operator, aka the json_extract function, see
|
180
223
|
https://duckdb.org/docs/extensions/json#json-extraction-functions
|
181
224
|
|
182
|
-
This works for Snowflake arrays too because we convert them to JSON
|
225
|
+
This works for Snowflake arrays too because we convert them to JSON in duckdb.
|
183
226
|
"""
|
184
227
|
if (
|
185
228
|
isinstance(expression, exp.Bracket)
|
@@ -259,6 +302,20 @@ def integer_precision(expression: exp.Expression) -> exp.Expression:
|
|
259
302
|
return expression
|
260
303
|
|
261
304
|
|
305
|
+
def json_extract_as_varchar(expression: exp.Expression) -> exp.Expression:
|
306
|
+
"""Return raw unquoted string when casting json extraction to varchar.
|
307
|
+
|
308
|
+
This must run after indices_to_json_extract.
|
309
|
+
|
310
|
+
Duckdb uses the ->> operator, aka the json_extract_string function, see
|
311
|
+
https://duckdb.org/docs/extensions/json#json-extraction-functions
|
312
|
+
"""
|
313
|
+
if isinstance(expression, exp.Cast) and (extract := expression.this) and isinstance(extract, exp.JSONExtract):
|
314
|
+
return exp.JSONExtractScalar(this=extract.this, expression=extract.expression)
|
315
|
+
|
316
|
+
return expression
|
317
|
+
|
318
|
+
|
262
319
|
def object_construct(expression: exp.Expression) -> exp.Expression:
|
263
320
|
"""Convert object_construct to return a json string
|
264
321
|
|
@@ -355,9 +412,13 @@ def regex_substr(expression: exp.Expression) -> exp.Expression:
|
|
355
412
|
|
356
413
|
# which occurrence of the pattern to match
|
357
414
|
try:
|
358
|
-
occurrence = expression.args["occurrence"]
|
415
|
+
occurrence = int(expression.args["occurrence"].this)
|
359
416
|
except KeyError:
|
360
|
-
occurrence =
|
417
|
+
occurrence = 1
|
418
|
+
|
419
|
+
# the duckdb dialect increments bracket (ie: index) expressions by 1 because duckdb is 1-indexed,
|
420
|
+
# so we need to compensate by subtracting 1
|
421
|
+
occurrence = exp.Literal(this=str(occurrence - 1), is_string=False)
|
361
422
|
|
362
423
|
try:
|
363
424
|
regex_parameters_value = str(expression.args["parameters"].this)
|
@@ -428,7 +489,7 @@ def set_schema(expression: exp.Expression, current_database: str | None) -> exp.
|
|
428
489
|
name = f"{expression.this.name}.main"
|
429
490
|
else:
|
430
491
|
# SCHEMA
|
431
|
-
if db := expression.this.args.get("db"):
|
492
|
+
if db := expression.this.args.get("db"): # noqa: SIM108
|
432
493
|
db_name = db.name
|
433
494
|
else:
|
434
495
|
# isn't qualified with a database
|
@@ -578,15 +639,14 @@ def semi_structured_types(expression: exp.Expression) -> exp.Expression:
|
|
578
639
|
exp.Expression: The transformed expression.
|
579
640
|
"""
|
580
641
|
|
581
|
-
if isinstance(expression, exp.DataType)
|
582
|
-
|
583
|
-
|
584
|
-
|
585
|
-
|
586
|
-
|
587
|
-
|
588
|
-
|
589
|
-
return new
|
642
|
+
if isinstance(expression, exp.DataType) and expression.this in [
|
643
|
+
exp.DataType.Type.ARRAY,
|
644
|
+
exp.DataType.Type.OBJECT,
|
645
|
+
exp.DataType.Type.VARIANT,
|
646
|
+
]:
|
647
|
+
new = expression.copy()
|
648
|
+
new.args["this"] = exp.DataType.Type.JSON
|
649
|
+
return new
|
590
650
|
|
591
651
|
return expression
|
592
652
|
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: fakesnow
|
3
|
-
Version: 0.
|
3
|
+
Version: 0.8.1
|
4
4
|
Summary: Fake Snowflake Connector for Python. Run Snowflake DB locally.
|
5
5
|
License: MIT License
|
6
6
|
|
@@ -32,7 +32,7 @@ License-File: LICENSE
|
|
32
32
|
Requires-Dist: duckdb~=0.9.2
|
33
33
|
Requires-Dist: pyarrow
|
34
34
|
Requires-Dist: snowflake-connector-python
|
35
|
-
Requires-Dist: sqlglot~=
|
35
|
+
Requires-Dist: sqlglot~=20.5.0
|
36
36
|
Provides-Extra: dev
|
37
37
|
Requires-Dist: black~=23.9; extra == "dev"
|
38
38
|
Requires-Dist: build~=1.0; extra == "dev"
|
@@ -63,6 +63,24 @@ pip install fakesnow
|
|
63
63
|
|
64
64
|
## Usage
|
65
65
|
|
66
|
+
Run script.py with fakesnow:
|
67
|
+
|
68
|
+
```shell
|
69
|
+
fakesnow script.py
|
70
|
+
```
|
71
|
+
|
72
|
+
Or pytest
|
73
|
+
|
74
|
+
```shell
|
75
|
+
fakesnow -m pytest
|
76
|
+
```
|
77
|
+
|
78
|
+
`fakesnow` executes `fakesnow.patch` before running the script or module.
|
79
|
+
|
80
|
+
### fakesnow.patch
|
81
|
+
|
82
|
+
eg:
|
83
|
+
|
66
84
|
```python
|
67
85
|
import fakesnow
|
68
86
|
import snowflake.connector
|
@@ -91,6 +109,8 @@ with fakesnow.patch("mymodule.write_pandas"):
|
|
91
109
|
...
|
92
110
|
```
|
93
111
|
|
112
|
+
### pytest fixtures
|
113
|
+
|
94
114
|
pytest [fixtures](fakesnow/fixtures.py) are provided for testing. Example _conftest.py_:
|
95
115
|
|
96
116
|
```python
|
@@ -146,7 +166,8 @@ For more detail see [tests/test_fakes.py](tests/test_fakes.py)
|
|
146
166
|
|
147
167
|
## Caveats
|
148
168
|
|
149
|
-
-
|
169
|
+
- The order of rows is non deterministic and may not match Snowflake unless ORDER BY is fully specified.
|
170
|
+
- VARCHAR field sizes are not enforced. Unlike Snowflake which errors with "User character length limit (xxx) exceeded by string" when an inserted string the exceeds the column limit.
|
150
171
|
|
151
172
|
## Contributing
|
152
173
|
|
@@ -3,7 +3,9 @@ MANIFEST.in
|
|
3
3
|
README.md
|
4
4
|
pyproject.toml
|
5
5
|
fakesnow/__init__.py
|
6
|
+
fakesnow/__main__.py
|
6
7
|
fakesnow/checks.py
|
8
|
+
fakesnow/cli.py
|
7
9
|
fakesnow/expr.py
|
8
10
|
fakesnow/fakes.py
|
9
11
|
fakesnow/fixtures.py
|
@@ -13,9 +15,11 @@ fakesnow/transforms.py
|
|
13
15
|
fakesnow.egg-info/PKG-INFO
|
14
16
|
fakesnow.egg-info/SOURCES.txt
|
15
17
|
fakesnow.egg-info/dependency_links.txt
|
18
|
+
fakesnow.egg-info/entry_points.txt
|
16
19
|
fakesnow.egg-info/requires.txt
|
17
20
|
fakesnow.egg-info/top_level.txt
|
18
21
|
tests/test_checks.py
|
22
|
+
tests/test_cli.py
|
19
23
|
tests/test_expr.py
|
20
24
|
tests/test_fakes.py
|
21
25
|
tests/test_patch.py
|
@@ -1,7 +1,7 @@
|
|
1
1
|
[project]
|
2
2
|
name = "fakesnow"
|
3
3
|
description = "Fake Snowflake Connector for Python. Run Snowflake DB locally."
|
4
|
-
version = "0.
|
4
|
+
version = "0.8.1"
|
5
5
|
readme = "README.md"
|
6
6
|
license = { file = "LICENSE" }
|
7
7
|
classifiers = ["License :: OSI Approved :: MIT License"]
|
@@ -11,12 +11,15 @@ dependencies = [
|
|
11
11
|
"duckdb~=0.9.2",
|
12
12
|
"pyarrow",
|
13
13
|
"snowflake-connector-python",
|
14
|
-
"sqlglot~=
|
14
|
+
"sqlglot~=20.5.0",
|
15
15
|
]
|
16
16
|
|
17
17
|
[project.urls]
|
18
18
|
homepage = "https://github.com/tekumara/fakesnow"
|
19
19
|
|
20
|
+
[project.scripts]
|
21
|
+
fakesnow = "fakesnow.cli:main"
|
22
|
+
|
20
23
|
[project.optional-dependencies]
|
21
24
|
dev = [
|
22
25
|
"black~=23.9",
|
@@ -81,6 +84,8 @@ select = [
|
|
81
84
|
"PERF",
|
82
85
|
# ruff-specific
|
83
86
|
"RUF",
|
87
|
+
# flake8-simplify
|
88
|
+
"SIM"
|
84
89
|
]
|
85
90
|
ignore = [
|
86
91
|
# allow untyped self and cls args, and no return type from dunder methods
|
@@ -0,0 +1,17 @@
|
|
1
|
+
from pytest import CaptureFixture
|
2
|
+
|
3
|
+
import fakesnow.cli
|
4
|
+
|
5
|
+
|
6
|
+
def test_run_module(capsys: CaptureFixture) -> None:
|
7
|
+
fakesnow.cli.main(["pytest", "-m", "tests.hello", "frobnitz"])
|
8
|
+
|
9
|
+
captured = capsys.readouterr()
|
10
|
+
assert captured.out == "('Hello fake frobnitz!',)\n"
|
11
|
+
|
12
|
+
|
13
|
+
def test_run_path(capsys: CaptureFixture) -> None:
|
14
|
+
fakesnow.cli.main(["pytest", "tests/hello.py", "frobnitz"])
|
15
|
+
|
16
|
+
captured = capsys.readouterr()
|
17
|
+
assert captured.out == "('Hello fake frobnitz!',)\n"
|
@@ -2,6 +2,7 @@
|
|
2
2
|
|
3
3
|
import datetime
|
4
4
|
import json
|
5
|
+
from collections.abc import Sequence
|
5
6
|
from decimal import Decimal
|
6
7
|
|
7
8
|
import pandas as pd
|
@@ -49,6 +50,10 @@ def test_binding_qmark(conn: snowflake.connector.SnowflakeConnection):
|
|
49
50
|
assert cur.fetchall() == [(1, "Jenny", True)]
|
50
51
|
|
51
52
|
|
53
|
+
def test_close(cur: snowflake.connector.cursor.SnowflakeCursor):
|
54
|
+
assert cur.close() is True
|
55
|
+
|
56
|
+
|
52
57
|
def test_connect_auto_create(_fakesnow: None):
|
53
58
|
with snowflake.connector.connect(database="db1", schema="schema1"):
|
54
59
|
# creates db1 and schema1
|
@@ -205,7 +210,7 @@ def test_describe(cur: snowflake.connector.cursor.SnowflakeCursor):
|
|
205
210
|
XINT INT, XINTEGER INTEGER, XBIGINT BIGINT, XSMALLINT SMALLINT, XTINYINT TINYINT, XBYTEINT BYTEINT,
|
206
211
|
XVARCHAR20 VARCHAR(20), XVARCHAR VARCHAR, XTEXT TEXT,
|
207
212
|
XTIMESTAMP TIMESTAMP, XTIMESTAMP_NTZ9 TIMESTAMP_NTZ(9), XDATE DATE, XTIME TIME,
|
208
|
-
XBINARY BINARY, XARRAY ARRAY, XOBJECT OBJECT
|
213
|
+
XBINARY BINARY, /* XARRAY ARRAY, XOBJECT OBJECT */ XVARIANT VARIANT
|
209
214
|
)
|
210
215
|
"""
|
211
216
|
)
|
@@ -233,8 +238,10 @@ def test_describe(cur: snowflake.connector.cursor.SnowflakeCursor):
|
|
233
238
|
ResultMetadata(name='XDATE', type_code=3, display_size=None, internal_size=None, precision=None, scale=None, is_nullable=True),
|
234
239
|
ResultMetadata(name='XTIME', type_code=12, display_size=None, internal_size=None, precision=0, scale=9, is_nullable=True),
|
235
240
|
ResultMetadata(name='XBINARY', type_code=11, display_size=None, internal_size=8388608, precision=None, scale=None, is_nullable=True),
|
236
|
-
|
237
|
-
ResultMetadata(name='
|
241
|
+
# TODO: handle ARRAY and OBJECT see https://github.com/tekumara/fakesnow/issues/26
|
242
|
+
# ResultMetadata(name='XARRAY', type_code=10, display_size=None, internal_size=16777216, precision=None, scale=None, is_nullable=True),
|
243
|
+
# ResultMetadata(name='XOBJECT', type_code=9, display_size=None, internal_size=None, precision=None, scale=None, is_nullable=True),
|
244
|
+
ResultMetadata(name='XVARIANT', type_code=5, display_size=None, internal_size=None, precision=None, scale=None, is_nullable=True),
|
238
245
|
]
|
239
246
|
# fmt: on
|
240
247
|
|
@@ -247,6 +254,19 @@ def test_describe(cur: snowflake.connector.cursor.SnowflakeCursor):
|
|
247
254
|
cur.execute("select * from example where XNUMBER = %s", (1,))
|
248
255
|
assert cur.description == expected_metadata
|
249
256
|
|
257
|
+
# test semi-structured ops return variant ie: type_code=5
|
258
|
+
# fmt: off
|
259
|
+
assert (
|
260
|
+
cur.describe("SELECT ['A', 'B'][0] as array_index, OBJECT_CONSTRUCT('k','v1')['k'] as object_key, ARRAY_CONSTRUCT('foo')::VARIANT[0] as variant_key")
|
261
|
+
== [
|
262
|
+
# NB: snowflake returns internal_size = 16777216 for all columns
|
263
|
+
ResultMetadata(name="ARRAY_INDEX", type_code=5, display_size=None, internal_size=None, precision=None, scale=None, is_nullable=True),
|
264
|
+
ResultMetadata(name="OBJECT_KEY", type_code=5, display_size=None, internal_size=None, precision=None, scale=None, is_nullable=True),
|
265
|
+
ResultMetadata(name="VARIANT_KEY", type_code=5, display_size=None, internal_size=None, precision=None, scale=None, is_nullable=True)
|
266
|
+
]
|
267
|
+
)
|
268
|
+
# fmt: on
|
269
|
+
|
250
270
|
|
251
271
|
def test_describe_info_schema_columns(cur: snowflake.connector.cursor.SnowflakeCursor):
|
252
272
|
# test we can handle the column types returned from the info schema, which are created by duckdb
|
@@ -262,6 +282,45 @@ def test_describe_info_schema_columns(cur: snowflake.connector.cursor.SnowflakeC
|
|
262
282
|
assert cur.description == expected_metadata
|
263
283
|
|
264
284
|
|
285
|
+
## descriptions are needed for ipython-sql/jupysql which describes every statement
|
286
|
+
|
287
|
+
|
288
|
+
def test_description_create_database(dcur: snowflake.connector.cursor.DictCursor):
|
289
|
+
dcur.execute("create database example")
|
290
|
+
assert dcur.fetchall() == [{"status": "Database EXAMPLE successfully created."}]
|
291
|
+
assert dcur.description == [ResultMetadata(name='status', type_code=2, display_size=None, internal_size=16777216, precision=None, scale=None, is_nullable=True)] # fmt: skip
|
292
|
+
# TODO: support drop database
|
293
|
+
# dcur.execute("drop database example")
|
294
|
+
# assert dcur.fetchall() == [{"status": "EXAMPLE successfully dropped."}]
|
295
|
+
# assert dcur.description == [ResultMetadata(name='status', type_code=2, display_size=None, internal_size=16777216, precision=None, scale=None, is_nullable=True)] # fmt: skip
|
296
|
+
|
297
|
+
|
298
|
+
def test_description_create_drop_schema(dcur: snowflake.connector.cursor.DictCursor):
|
299
|
+
dcur.execute("create schema example")
|
300
|
+
assert dcur.fetchall() == [{"status": "Schema EXAMPLE successfully created."}]
|
301
|
+
assert dcur.description == [ResultMetadata(name='status', type_code=2, display_size=None, internal_size=16777216, precision=None, scale=None, is_nullable=True)] # fmt: skip
|
302
|
+
dcur.execute("drop schema example")
|
303
|
+
assert dcur.fetchall() == [{"status": "EXAMPLE successfully dropped."}]
|
304
|
+
assert dcur.description == [ResultMetadata(name='status', type_code=2, display_size=None, internal_size=16777216, precision=None, scale=None, is_nullable=True)] # fmt: skip
|
305
|
+
|
306
|
+
|
307
|
+
def test_description_create_drop_table(dcur: snowflake.connector.cursor.DictCursor):
|
308
|
+
dcur.execute("create table example (X int)")
|
309
|
+
assert dcur.fetchall() == [{"status": "Table EXAMPLE successfully created."}]
|
310
|
+
assert dcur.description == [ResultMetadata(name='status', type_code=2, display_size=None, internal_size=16777216, precision=None, scale=None, is_nullable=True)] # fmt: skip
|
311
|
+
dcur.execute("drop table example")
|
312
|
+
assert dcur.fetchall() == [{"status": "EXAMPLE successfully dropped."}]
|
313
|
+
assert dcur.description == [ResultMetadata(name='status', type_code=2, display_size=None, internal_size=16777216, precision=None, scale=None, is_nullable=True)] # fmt: skip
|
314
|
+
|
315
|
+
|
316
|
+
def test_description_insert(dcur: snowflake.connector.cursor.DictCursor):
|
317
|
+
dcur.execute("create table example (X int)")
|
318
|
+
dcur.execute("insert into example values (1), (2)")
|
319
|
+
assert dcur.fetchall() == [{"number of rows inserted": 2}]
|
320
|
+
# TODO: Snowflake is actually precision=19, is_nullable=False
|
321
|
+
assert dcur.description == [ResultMetadata(name='number of rows inserted', type_code=0, display_size=None, internal_size=None, precision=38, scale=0, is_nullable=True)] # fmt: skip
|
322
|
+
|
323
|
+
|
265
324
|
def test_executemany(cur: snowflake.connector.cursor.SnowflakeCursor):
|
266
325
|
cur.execute("create table customers (ID int, FIRST_NAME varchar, LAST_NAME varchar)")
|
267
326
|
|
@@ -280,49 +339,65 @@ def test_execute_string(conn: snowflake.connector.SnowflakeConnection):
|
|
280
339
|
assert [(1,)] == cur2.fetchall()
|
281
340
|
|
282
341
|
|
283
|
-
def test_fetchall(
|
284
|
-
|
285
|
-
cur.execute("insert into customers values (1, 'Jenny', 'P')")
|
286
|
-
cur.execute("insert into customers values (2, 'Jasper', 'M')")
|
287
|
-
cur.execute("select id, first_name, last_name from customers")
|
288
|
-
|
289
|
-
assert cur.fetchall() == [(1, "Jenny", "P"), (2, "Jasper", "M")]
|
290
|
-
|
291
|
-
|
292
|
-
def test_fetchall_dict_cursor(conn: snowflake.connector.SnowflakeConnection):
|
293
|
-
with conn.cursor(snowflake.connector.cursor.DictCursor) as cur:
|
342
|
+
def test_fetchall(conn: snowflake.connector.SnowflakeConnection):
|
343
|
+
with conn.cursor() as cur:
|
294
344
|
cur.execute("create table customers (ID int, FIRST_NAME varchar, LAST_NAME varchar)")
|
295
345
|
cur.execute("insert into customers values (1, 'Jenny', 'P')")
|
296
346
|
cur.execute("insert into customers values (2, 'Jasper', 'M')")
|
297
347
|
cur.execute("select id, first_name, last_name from customers")
|
298
348
|
|
349
|
+
assert cur.fetchall() == [(1, "Jenny", "P"), (2, "Jasper", "M")]
|
350
|
+
|
351
|
+
with conn.cursor(snowflake.connector.cursor.DictCursor) as cur:
|
352
|
+
cur.execute("select id, first_name, last_name from customers")
|
353
|
+
|
299
354
|
assert cur.fetchall() == [
|
300
355
|
{"ID": 1, "FIRST_NAME": "Jenny", "LAST_NAME": "P"},
|
301
356
|
{"ID": 2, "FIRST_NAME": "Jasper", "LAST_NAME": "M"},
|
302
357
|
]
|
303
358
|
|
304
359
|
|
305
|
-
def test_fetchone(
|
306
|
-
|
307
|
-
|
308
|
-
|
309
|
-
|
310
|
-
|
311
|
-
assert cur.fetchone() == (1, "Jenny", "P")
|
312
|
-
assert cur.fetchone() == (2, "Jasper", "M")
|
313
|
-
assert not cur.fetchone()
|
360
|
+
def test_fetchone(conn: snowflake.connector.SnowflakeConnection):
|
361
|
+
with conn.cursor() as cur:
|
362
|
+
cur.execute("create table customers (ID int, FIRST_NAME varchar, LAST_NAME varchar)")
|
363
|
+
cur.execute("insert into customers values (1, 'Jenny', 'P')")
|
364
|
+
cur.execute("insert into customers values (2, 'Jasper', 'M')")
|
365
|
+
cur.execute("select id, first_name, last_name from customers")
|
314
366
|
|
367
|
+
assert cur.fetchone() == (1, "Jenny", "P")
|
368
|
+
assert cur.fetchone() == (2, "Jasper", "M")
|
369
|
+
assert cur.fetchone() is None
|
315
370
|
|
316
|
-
def test_fetchone_dict_cursor(conn: snowflake.connector.SnowflakeConnection):
|
317
371
|
with conn.cursor(snowflake.connector.cursor.DictCursor) as cur:
|
372
|
+
cur.execute("select id, first_name, last_name from customers")
|
373
|
+
|
374
|
+
assert cur.fetchone() == {"ID": 1, "FIRST_NAME": "Jenny", "LAST_NAME": "P"}
|
375
|
+
assert cur.fetchone() == {"ID": 2, "FIRST_NAME": "Jasper", "LAST_NAME": "M"}
|
376
|
+
assert cur.fetchone() is None
|
377
|
+
|
378
|
+
|
379
|
+
def test_fetchmany(conn: snowflake.connector.SnowflakeConnection):
|
380
|
+
with conn.cursor() as cur:
|
318
381
|
cur.execute("create table customers (ID int, FIRST_NAME varchar, LAST_NAME varchar)")
|
319
382
|
cur.execute("insert into customers values (1, 'Jenny', 'P')")
|
320
383
|
cur.execute("insert into customers values (2, 'Jasper', 'M')")
|
384
|
+
cur.execute("insert into customers values (3, 'Jeremy', 'K')")
|
321
385
|
cur.execute("select id, first_name, last_name from customers")
|
322
386
|
|
323
|
-
assert cur.
|
324
|
-
assert cur.
|
325
|
-
assert
|
387
|
+
assert cur.fetchmany(2) == [(1, "Jenny", "P"), (2, "Jasper", "M")]
|
388
|
+
assert cur.fetchmany(2) == [(3, "Jeremy", "K")]
|
389
|
+
assert cur.fetchmany(2) == []
|
390
|
+
|
391
|
+
with conn.cursor(snowflake.connector.cursor.DictCursor) as cur:
|
392
|
+
cur.execute("select id, first_name, last_name from customers")
|
393
|
+
assert cur.fetchmany(2) == [
|
394
|
+
{"ID": 1, "FIRST_NAME": "Jenny", "LAST_NAME": "P"},
|
395
|
+
{"ID": 2, "FIRST_NAME": "Jasper", "LAST_NAME": "M"},
|
396
|
+
]
|
397
|
+
assert cur.fetchmany(2) == [
|
398
|
+
{"ID": 3, "FIRST_NAME": "Jeremy", "LAST_NAME": "K"},
|
399
|
+
]
|
400
|
+
assert cur.fetchmany(2) == []
|
326
401
|
|
327
402
|
|
328
403
|
def test_fetch_pandas_all(cur: snowflake.connector.cursor.SnowflakeCursor):
|
@@ -341,6 +416,23 @@ def test_fetch_pandas_all(cur: snowflake.connector.cursor.SnowflakeCursor):
|
|
341
416
|
assert_frame_equal(cur.fetch_pandas_all(), expected_df)
|
342
417
|
|
343
418
|
|
419
|
+
def test_flatten(cur: snowflake.connector.cursor.SnowflakeCursor):
|
420
|
+
cur.execute(
|
421
|
+
"""
|
422
|
+
select t.id, flat.value:fruit from
|
423
|
+
(
|
424
|
+
select 1, parse_json('[{"fruit":"banana"}]')
|
425
|
+
union
|
426
|
+
select 2, parse_json('[{"fruit":"coconut"}, {"fruit":"durian"}]')
|
427
|
+
) as t(id, fruits), lateral flatten(input => t.fruits) AS flat
|
428
|
+
order by id
|
429
|
+
"""
|
430
|
+
# duckdb lateral join order is non-deterministic so order by id
|
431
|
+
# within an id the order of fruits should match the json array
|
432
|
+
)
|
433
|
+
assert cur.fetchall() == [(1, '"banana"'), (2, '"coconut"'), (2, '"durian"')]
|
434
|
+
|
435
|
+
|
344
436
|
def test_floats_are_64bit(cur: snowflake.connector.cursor.SnowflakeCursor):
|
345
437
|
cur.execute("create or replace table example (f float, f4 float4, f8 float8, d double, r real)")
|
346
438
|
cur.execute("insert into example values (1.23, 1.23, 1.23, 1.23, 1.23)")
|
@@ -349,6 +441,12 @@ def test_floats_are_64bit(cur: snowflake.connector.cursor.SnowflakeCursor):
|
|
349
441
|
assert cur.fetchall() == [(1.23, 1.23, 1.23, 1.23, 1.23)]
|
350
442
|
|
351
443
|
|
444
|
+
def test_get_path_as_varchar(cur: snowflake.connector.cursor.SnowflakeCursor):
|
445
|
+
cur.execute("""select parse_json('{"fruit":"banana"}'):fruit::varchar""")
|
446
|
+
# converting json to varchar returns unquoted string
|
447
|
+
assert cur.fetchall() == [("banana",)]
|
448
|
+
|
449
|
+
|
352
450
|
def test_get_result_batches(cur: snowflake.connector.cursor.SnowflakeCursor):
|
353
451
|
cur.execute("create table customers (ID int, FIRST_NAME varchar, LAST_NAME varchar)")
|
354
452
|
cur.execute("insert into customers values (1, 'Jenny', 'P')")
|
@@ -420,7 +518,7 @@ def test_information_schema_columns_other(cur: snowflake.connector.cursor.Snowfl
|
|
420
518
|
"""
|
421
519
|
create or replace table example (
|
422
520
|
XTIMESTAMP TIMESTAMP, XTIMESTAMP_NTZ9 TIMESTAMP_NTZ(9), XDATE DATE, XTIME TIME,
|
423
|
-
XBINARY BINARY, XARRAY ARRAY, XOBJECT OBJECT
|
521
|
+
XBINARY BINARY, /* XARRAY ARRAY, XOBJECT OBJECT */ XVARIANT VARIANT
|
424
522
|
)
|
425
523
|
"""
|
426
524
|
)
|
@@ -438,8 +536,10 @@ def test_information_schema_columns_other(cur: snowflake.connector.cursor.Snowfl
|
|
438
536
|
("XDATE", "DATE"),
|
439
537
|
("XTIME", "TIME"),
|
440
538
|
("XBINARY", "BINARY"),
|
441
|
-
|
442
|
-
("
|
539
|
+
# TODO: support these types https://github.com/tekumara/fakesnow/issues/27
|
540
|
+
# ("XARRAY", "ARRAY"),
|
541
|
+
# ("XOBJECT", "OBJECT"),
|
542
|
+
("XVARIANT", "VARIANT"),
|
443
543
|
]
|
444
544
|
|
445
545
|
|
@@ -547,18 +647,25 @@ def test_schema_drop(cur: snowflake.connector.cursor.SnowflakeCursor):
|
|
547
647
|
|
548
648
|
|
549
649
|
def test_semi_structured_types(cur: snowflake.connector.cursor.SnowflakeCursor):
|
550
|
-
|
650
|
+
def indent(rows: Sequence[tuple]) -> list[tuple]:
|
651
|
+
# indent duckdb json strings to match snowflake json strings
|
652
|
+
return [(json.dumps(json.loads(r[0]), indent=2), *r[1:]) for r in rows]
|
653
|
+
|
654
|
+
cur.execute("create or replace table semis (emails array, name object, notes variant)")
|
551
655
|
cur.execute(
|
552
|
-
"""insert into semis(emails, name, notes) SELECT [
|
656
|
+
"""insert into semis(emails, name, notes) SELECT ['A', 'B'], OBJECT_CONSTRUCT('k','v1'), ARRAY_CONSTRUCT('foo')::VARIANT"""
|
553
657
|
)
|
554
658
|
cur.execute(
|
555
|
-
"""insert into semis(emails, name, notes)
|
659
|
+
"""insert into semis(emails, name, notes) SELECT ['C','D'], parse_json('{"k": "v2"}'), parse_json('{"b": "ar"}')"""
|
556
660
|
)
|
557
661
|
|
558
662
|
# results are returned as strings, because the underlying type is JSON (duckdb) / VARIANT (snowflake)
|
559
663
|
|
664
|
+
cur.execute("select emails from semis")
|
665
|
+
assert indent(cur.fetchall()) == [('[\n "A",\n "B"\n]',), ('[\n "C",\n "D"\n]',)] # type: ignore
|
666
|
+
|
560
667
|
cur.execute("select emails[0] from semis")
|
561
|
-
assert cur.fetchall() == [("
|
668
|
+
assert cur.fetchall() == [('"A"',), ('"C"',)]
|
562
669
|
|
563
670
|
cur.execute("select name['k'] from semis")
|
564
671
|
assert cur.fetchall() == [('"v1"',), ('"v2"',)]
|
@@ -632,21 +739,6 @@ def test_to_decimal(cur: snowflake.connector.cursor.SnowflakeCursor):
|
|
632
739
|
]
|
633
740
|
|
634
741
|
|
635
|
-
def test_write_pandas_timestamp_ntz(conn: snowflake.connector.SnowflakeConnection):
|
636
|
-
# compensate for https://github.com/duckdb/duckdb/issues/7980
|
637
|
-
with conn.cursor() as cur:
|
638
|
-
cur.execute("create table example (UPDATE_AT_NTZ timestamp_ntz(9))")
|
639
|
-
# cur.execute("create table example (UPDATE_AT_NTZ timestamp)")
|
640
|
-
|
641
|
-
now_utc = datetime.datetime.now(pytz.utc)
|
642
|
-
df = pd.DataFrame([(now_utc,)], columns=["UPDATE_AT_NTZ"])
|
643
|
-
snowflake.connector.pandas_tools.write_pandas(conn, df, "EXAMPLE")
|
644
|
-
|
645
|
-
cur.execute("select * from example")
|
646
|
-
|
647
|
-
assert cur.fetchall() == [(now_utc.replace(tzinfo=None),)]
|
648
|
-
|
649
|
-
|
650
742
|
def test_transactions(conn: snowflake.connector.SnowflakeConnection):
|
651
743
|
conn.execute_string(
|
652
744
|
"""CREATE TABLE table1 (i int);
|
@@ -663,6 +755,17 @@ def test_transactions(conn: snowflake.connector.SnowflakeConnection):
|
|
663
755
|
cur.execute("SELECT * FROM table1")
|
664
756
|
assert cur.fetchall() == [(2,)]
|
665
757
|
|
758
|
+
# check rollback and commit without transaction is a success (to mimic snowflake)
|
759
|
+
# also check description can be retrieved, needed for ipython-sql/jupysql which runs description implicitly
|
760
|
+
with conn.cursor() as cur:
|
761
|
+
cur.execute("COMMIT")
|
762
|
+
assert cur.description == [ResultMetadata(name='status', type_code=2, display_size=None, internal_size=16777216, precision=None, scale=None, is_nullable=True)] # fmt: skip
|
763
|
+
assert cur.fetchall() == [("Statement executed successfully.",)]
|
764
|
+
|
765
|
+
cur.execute("ROLLBACK")
|
766
|
+
assert cur.description == [ResultMetadata(name='status', type_code=2, display_size=None, internal_size=16777216, precision=None, scale=None, is_nullable=True)] # fmt: skip
|
767
|
+
assert cur.fetchall() == [("Statement executed successfully.",)]
|
768
|
+
|
666
769
|
|
667
770
|
def test_unquoted_identifiers_are_upper_cased(conn: snowflake.connector.SnowflakeConnection):
|
668
771
|
with conn.cursor(snowflake.connector.cursor.DictCursor) as cur:
|
@@ -728,12 +831,12 @@ def test_values(conn: snowflake.connector.SnowflakeConnection):
|
|
728
831
|
|
729
832
|
def test_write_pandas(conn: snowflake.connector.SnowflakeConnection):
|
730
833
|
with conn.cursor() as cur:
|
731
|
-
cur.execute("create table customers (ID int, FIRST_NAME varchar, LAST_NAME varchar)")
|
834
|
+
cur.execute("create table customers (ID int, FIRST_NAME varchar, LAST_NAME varchar, ORDERS array)")
|
732
835
|
|
733
836
|
df = pd.DataFrame.from_records(
|
734
837
|
[
|
735
|
-
{"ID": 1, "FIRST_NAME": "Jenny", "LAST_NAME": "P"},
|
736
|
-
{"ID": 2, "FIRST_NAME": "Jasper", "LAST_NAME": "M"},
|
838
|
+
{"ID": 1, "FIRST_NAME": "Jenny", "LAST_NAME": "P", "ORDERS": ["A", "B"]},
|
839
|
+
{"ID": 2, "FIRST_NAME": "Jasper", "LAST_NAME": "M", "ORDERS": ["C", "D"]},
|
737
840
|
]
|
738
841
|
)
|
739
842
|
snowflake.connector.pandas_tools.write_pandas(conn, df, "customers")
|
@@ -743,6 +846,21 @@ def test_write_pandas(conn: snowflake.connector.SnowflakeConnection):
|
|
743
846
|
assert cur.fetchall() == [(1, "Jenny", "P"), (2, "Jasper", "M")]
|
744
847
|
|
745
848
|
|
849
|
+
def test_write_pandas_timestamp_ntz(conn: snowflake.connector.SnowflakeConnection):
|
850
|
+
# compensate for https://github.com/duckdb/duckdb/issues/7980
|
851
|
+
with conn.cursor() as cur:
|
852
|
+
cur.execute("create table example (UPDATE_AT_NTZ timestamp_ntz(9))")
|
853
|
+
# cur.execute("create table example (UPDATE_AT_NTZ timestamp)")
|
854
|
+
|
855
|
+
now_utc = datetime.datetime.now(pytz.utc)
|
856
|
+
df = pd.DataFrame([(now_utc,)], columns=["UPDATE_AT_NTZ"])
|
857
|
+
snowflake.connector.pandas_tools.write_pandas(conn, df, "EXAMPLE")
|
858
|
+
|
859
|
+
cur.execute("select * from example")
|
860
|
+
|
861
|
+
assert cur.fetchall() == [(now_utc.replace(tzinfo=None),)]
|
862
|
+
|
863
|
+
|
746
864
|
def test_write_pandas_partial_columns(conn: snowflake.connector.SnowflakeConnection):
|
747
865
|
with conn.cursor() as cur:
|
748
866
|
cur.execute("create table customers (ID int, FIRST_NAME varchar, LAST_NAME varchar)")
|
@@ -39,7 +39,7 @@ def test_patch_other_unloaded_module() -> None:
|
|
39
39
|
def test_cannot_patch_twice(_fakesnow_no_auto_create: None) -> None:
|
40
40
|
# _fakesnow is the first patch
|
41
41
|
|
42
|
-
with pytest.raises(AssertionError) as excinfo:
|
42
|
+
with pytest.raises(AssertionError) as excinfo: # noqa: SIM117
|
43
43
|
# second patch will fail
|
44
44
|
with fakesnow.patch():
|
45
45
|
pass
|
@@ -7,11 +7,13 @@ from fakesnow.transforms import (
|
|
7
7
|
drop_schema_cascade,
|
8
8
|
extract_comment,
|
9
9
|
extract_text_length,
|
10
|
+
flatten,
|
10
11
|
float_to_double,
|
11
12
|
indices_to_json_extract,
|
12
13
|
information_schema_columns_snowflake,
|
13
14
|
information_schema_tables_ext,
|
14
15
|
integer_precision,
|
16
|
+
json_extract_as_varchar,
|
15
17
|
object_construct,
|
16
18
|
parse_json,
|
17
19
|
regex_replace,
|
@@ -67,6 +69,25 @@ def test_extract_text_length() -> None:
|
|
67
69
|
assert e.args["text_lengths"] == [("t1", 16777216), ("t2", 10), ("t3", 20)]
|
68
70
|
|
69
71
|
|
72
|
+
def test_flatten() -> None:
|
73
|
+
assert (
|
74
|
+
sqlglot.parse_one(
|
75
|
+
"""
|
76
|
+
select t.id, flat.value:fruit from
|
77
|
+
(
|
78
|
+
select 1, parse_json('[{"fruit":"banana"}]')
|
79
|
+
union
|
80
|
+
select 2, parse_json('[{"fruit":"coconut"}, {"fruit":"durian"}]')
|
81
|
+
) as t(id, fruits), lateral flatten(input => t.fruits) AS flat
|
82
|
+
""",
|
83
|
+
read="snowflake",
|
84
|
+
).transform(flatten)
|
85
|
+
# needed to transform flat.value:fruit to flat.value -> '$.fruit'
|
86
|
+
.transform(indices_to_json_extract).sql(dialect="duckdb")
|
87
|
+
== """SELECT t.id, flat.value -> '$.fruit' FROM (SELECT 1, JSON('[{"fruit":"banana"}]') UNION SELECT 2, JSON('[{"fruit":"coconut"}, {"fruit":"durian"}]')) AS t(id, fruits), LATERAL UNNEST(LIST_REVERSE(CAST(t.fruits AS JSON[]))) AS flat(VALUE)""" # noqa: E501
|
88
|
+
)
|
89
|
+
|
90
|
+
|
70
91
|
def test_float_to_double() -> None:
|
71
92
|
assert (
|
72
93
|
sqlglot.parse_one("create table example (f float, f4 float4, f8 float8, d double, r real)")
|
@@ -121,6 +142,18 @@ def test_information_schema_tables_ext() -> None:
|
|
121
142
|
)
|
122
143
|
|
123
144
|
|
145
|
+
def test_json_extract_as_varchar() -> None:
|
146
|
+
assert (
|
147
|
+
sqlglot.parse_one(
|
148
|
+
"""select parse_json('{"fruit":"banana"}'):fruit::varchar""",
|
149
|
+
read="snowflake",
|
150
|
+
)
|
151
|
+
# needed first
|
152
|
+
.transform(indices_to_json_extract).transform(json_extract_as_varchar).sql(dialect="duckdb")
|
153
|
+
== """SELECT JSON('{"fruit":"banana"}') ->> '$.fruit'"""
|
154
|
+
)
|
155
|
+
|
156
|
+
|
124
157
|
def test_object_construct() -> None:
|
125
158
|
assert (
|
126
159
|
sqlglot.parse_one("SELECT OBJECT_CONSTRUCT('a',1,'b','BBBB', 'c',null)", read="snowflake")
|
@@ -150,7 +183,7 @@ def test_regex_substr() -> None:
|
|
150
183
|
assert (
|
151
184
|
sqlglot.parse_one("SELECT regexp_substr(string1, 'the\\\\W+\\\\w+')", read="snowflake")
|
152
185
|
.transform(regex_substr)
|
153
|
-
.sql()
|
186
|
+
.sql(dialect="duckdb")
|
154
187
|
== "SELECT REGEXP_EXTRACT_ALL(string1[1 : ], 'the\\W+\\w+', 0, '')[1]"
|
155
188
|
)
|
156
189
|
|
@@ -163,7 +196,7 @@ def test_semi_structured_types() -> None:
|
|
163
196
|
|
164
197
|
assert (
|
165
198
|
sqlglot.parse_one("CREATE TABLE table1 (name array)").transform(semi_structured_types).sql(dialect="duckdb")
|
166
|
-
== "CREATE TABLE table1 (name JSON
|
199
|
+
== "CREATE TABLE table1 (name JSON)"
|
167
200
|
)
|
168
201
|
|
169
202
|
assert (
|
@@ -188,7 +221,7 @@ def test_timestamp_ntz_ns() -> None:
|
|
188
221
|
def test_to_date() -> None:
|
189
222
|
assert (
|
190
223
|
sqlglot.parse_one("SELECT to_date(to_timestamp(0))").transform(to_date).sql()
|
191
|
-
== "SELECT CAST(DATE_TRUNC('
|
224
|
+
== "SELECT CAST(DATE_TRUNC('DAY', TO_TIMESTAMP(0)) AS DATE)"
|
192
225
|
)
|
193
226
|
|
194
227
|
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|