fakesnow 0.9.11__tar.gz → 0.9.13__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fakesnow-0.9.11 → fakesnow-0.9.13}/PKG-INFO +2 -2
- {fakesnow-0.9.11 → fakesnow-0.9.13}/fakesnow/__init__.py +6 -1
- {fakesnow-0.9.11 → fakesnow-0.9.13}/fakesnow/fakes.py +31 -12
- {fakesnow-0.9.11 → fakesnow-0.9.13}/fakesnow/transforms.py +2 -4
- {fakesnow-0.9.11 → fakesnow-0.9.13}/fakesnow.egg-info/PKG-INFO +2 -2
- {fakesnow-0.9.11 → fakesnow-0.9.13}/fakesnow.egg-info/SOURCES.txt +2 -1
- {fakesnow-0.9.11 → fakesnow-0.9.13}/fakesnow.egg-info/requires.txt +1 -1
- {fakesnow-0.9.11 → fakesnow-0.9.13}/pyproject.toml +2 -2
- {fakesnow-0.9.11 → fakesnow-0.9.13}/tests/test_fakes.py +7 -160
- {fakesnow-0.9.11 → fakesnow-0.9.13}/tests/test_transforms.py +4 -0
- fakesnow-0.9.13/tests/test_write_pandas.py +164 -0
- {fakesnow-0.9.11 → fakesnow-0.9.13}/LICENSE +0 -0
- {fakesnow-0.9.11 → fakesnow-0.9.13}/README.md +0 -0
- {fakesnow-0.9.11 → fakesnow-0.9.13}/fakesnow/__main__.py +0 -0
- {fakesnow-0.9.11 → fakesnow-0.9.13}/fakesnow/checks.py +0 -0
- {fakesnow-0.9.11 → fakesnow-0.9.13}/fakesnow/cli.py +0 -0
- {fakesnow-0.9.11 → fakesnow-0.9.13}/fakesnow/expr.py +0 -0
- {fakesnow-0.9.11 → fakesnow-0.9.13}/fakesnow/fixtures.py +0 -0
- {fakesnow-0.9.11 → fakesnow-0.9.13}/fakesnow/global_database.py +0 -0
- {fakesnow-0.9.11 → fakesnow-0.9.13}/fakesnow/info_schema.py +0 -0
- {fakesnow-0.9.11 → fakesnow-0.9.13}/fakesnow/macros.py +0 -0
- {fakesnow-0.9.11 → fakesnow-0.9.13}/fakesnow/py.typed +0 -0
- {fakesnow-0.9.11 → fakesnow-0.9.13}/fakesnow.egg-info/dependency_links.txt +0 -0
- {fakesnow-0.9.11 → fakesnow-0.9.13}/fakesnow.egg-info/entry_points.txt +0 -0
- {fakesnow-0.9.11 → fakesnow-0.9.13}/fakesnow.egg-info/top_level.txt +0 -0
- {fakesnow-0.9.11 → fakesnow-0.9.13}/setup.cfg +0 -0
- {fakesnow-0.9.11 → fakesnow-0.9.13}/tests/test_checks.py +0 -0
- {fakesnow-0.9.11 → fakesnow-0.9.13}/tests/test_cli.py +0 -0
- {fakesnow-0.9.11 → fakesnow-0.9.13}/tests/test_expr.py +0 -0
- {fakesnow-0.9.11 → fakesnow-0.9.13}/tests/test_info_schema.py +0 -0
- {fakesnow-0.9.11 → fakesnow-0.9.13}/tests/test_patch.py +0 -0
- {fakesnow-0.9.11 → fakesnow-0.9.13}/tests/test_sqlalchemy.py +0 -0
- {fakesnow-0.9.11 → fakesnow-0.9.13}/tests/test_users.py +0 -0
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: fakesnow
|
3
|
-
Version: 0.9.
|
3
|
+
Version: 0.9.13
|
4
4
|
Summary: Fake Snowflake Connector for Python. Run, mock and test Snowflake DB locally.
|
5
5
|
License: Apache License
|
6
6
|
Version 2.0, January 2004
|
@@ -213,7 +213,7 @@ License-File: LICENSE
|
|
213
213
|
Requires-Dist: duckdb~=0.10.0
|
214
214
|
Requires-Dist: pyarrow
|
215
215
|
Requires-Dist: snowflake-connector-python
|
216
|
-
Requires-Dist: sqlglot~=23.
|
216
|
+
Requires-Dist: sqlglot~=23.14.0
|
217
217
|
Provides-Extra: dev
|
218
218
|
Requires-Dist: build~=1.0; extra == "dev"
|
219
219
|
Requires-Dist: pandas-stubs; extra == "dev"
|
@@ -21,6 +21,7 @@ def patch(
|
|
21
21
|
create_database_on_connect: bool = True,
|
22
22
|
create_schema_on_connect: bool = True,
|
23
23
|
db_path: str | os.PathLike | None = None,
|
24
|
+
nop_regexes: list[str] | None = None,
|
24
25
|
) -> Iterator[None]:
|
25
26
|
"""Patch snowflake targets with fakes.
|
26
27
|
|
@@ -36,8 +37,11 @@ def patch(
|
|
36
37
|
|
37
38
|
create_database_on_connect (bool, optional): Create database if provided in connection. Defaults to True.
|
38
39
|
create_schema_on_connect (bool, optional): Create schema if provided in connection. Defaults to True.
|
39
|
-
db_path (str | os.PathLike | None, optional):
|
40
|
+
db_path (str | os.PathLike | None, optional): Use existing database files from this path
|
40
41
|
or create them here if they don't already exist. If None databases are in-memory. Defaults to None.
|
42
|
+
nop_regexes (list[str] | None, optional): SQL statements matching these regexes (case-insensitive) will return
|
43
|
+
the success response without being run. Useful to skip over SQL commands that aren't implemented yet.
|
44
|
+
Defaults to None.
|
41
45
|
|
42
46
|
Yields:
|
43
47
|
Iterator[None]: None.
|
@@ -57,6 +61,7 @@ def patch(
|
|
57
61
|
create_database=create_database_on_connect,
|
58
62
|
create_schema=create_schema_on_connect,
|
59
63
|
db_path=db_path,
|
64
|
+
nop_regexes=nop_regexes,
|
60
65
|
**kwargs,
|
61
66
|
),
|
62
67
|
snowflake.connector.pandas_tools.write_pandas: fakes.write_pandas,
|
@@ -16,6 +16,7 @@ from sqlglot import exp
|
|
16
16
|
if TYPE_CHECKING:
|
17
17
|
import pandas as pd
|
18
18
|
import pyarrow.lib
|
19
|
+
import numpy as np
|
19
20
|
import pyarrow
|
20
21
|
import snowflake.connector.converter
|
21
22
|
import snowflake.connector.errors
|
@@ -134,8 +135,11 @@ class FakeSnowflakeCursor:
|
|
134
135
|
print(f"{command};{params=}" if params else f"{command};", file=sys.stderr)
|
135
136
|
|
136
137
|
command, params = self._rewrite_with_params(command, params)
|
137
|
-
|
138
|
-
|
138
|
+
if self._conn.nop_regexes and any(re.match(p, command, re.IGNORECASE) for p in self._conn.nop_regexes):
|
139
|
+
transformed = transforms.SUCCESS_NOP
|
140
|
+
else:
|
141
|
+
expression = parse_one(command, read="snowflake")
|
142
|
+
transformed = self._transform(expression)
|
139
143
|
return self._execute(transformed, params)
|
140
144
|
except snowflake.connector.errors.ProgrammingError as e:
|
141
145
|
self._sqlstate = e.sqlstate
|
@@ -501,6 +505,7 @@ class FakeSnowflakeConnection:
|
|
501
505
|
create_database: bool = True,
|
502
506
|
create_schema: bool = True,
|
503
507
|
db_path: str | os.PathLike | None = None,
|
508
|
+
nop_regexes: list[str] | None = None,
|
504
509
|
*args: Any,
|
505
510
|
**kwargs: Any,
|
506
511
|
):
|
@@ -512,6 +517,7 @@ class FakeSnowflakeConnection:
|
|
512
517
|
self.database_set = False
|
513
518
|
self.schema_set = False
|
514
519
|
self.db_path = db_path
|
520
|
+
self.nop_regexes = nop_regexes
|
515
521
|
self._paramstyle = snowflake.connector.paramstyle
|
516
522
|
|
517
523
|
create_global_database(duck_conn)
|
@@ -606,9 +612,7 @@ class FakeSnowflakeConnection:
|
|
606
612
|
def rollback(self) -> None:
|
607
613
|
self.cursor().execute("ROLLBACK")
|
608
614
|
|
609
|
-
def _insert_df(
|
610
|
-
self, df: pd.DataFrame, table_name: str, database: str | None = None, schema: str | None = None
|
611
|
-
) -> int:
|
615
|
+
def _insert_df(self, df: pd.DataFrame, table_name: str) -> int:
|
612
616
|
# Objects in dataframes are written as parquet structs, and snowflake loads parquet structs as json strings.
|
613
617
|
# Whereas duckdb analyses a dataframe see https://duckdb.org/docs/api/python/data_ingestion.html#pandas-dataframes--object-columns
|
614
618
|
# and converts a object to the most specific type possible, eg: dict -> STRUCT, MAP or varchar, and list -> LIST
|
@@ -630,12 +634,7 @@ class FakeSnowflakeConnection:
|
|
630
634
|
df[col] = df[col].apply(lambda x: json.dumps(x) if isinstance(x, (dict, list)) else x)
|
631
635
|
|
632
636
|
escaped_cols = ",".join(f'"{col}"' for col in df.columns.to_list())
|
633
|
-
|
634
|
-
if schema:
|
635
|
-
table_name = f"{schema}.{table_name}"
|
636
|
-
if database:
|
637
|
-
name = f"{database}.{table_name}"
|
638
|
-
self._duck_conn.execute(f"INSERT INTO {name}({escaped_cols}) SELECT * FROM df")
|
637
|
+
self._duck_conn.execute(f"INSERT INTO {table_name}({escaped_cols}) SELECT * FROM df")
|
639
638
|
|
640
639
|
return self._duck_conn.fetchall()[0][0]
|
641
640
|
|
@@ -685,6 +684,15 @@ WritePandasResult = tuple[
|
|
685
684
|
]
|
686
685
|
|
687
686
|
|
687
|
+
def sql_type(dtype: np.dtype) -> str:
|
688
|
+
if str(dtype) == "int64":
|
689
|
+
return "NUMBER"
|
690
|
+
elif str(dtype) == "object":
|
691
|
+
return "VARCHAR"
|
692
|
+
else:
|
693
|
+
raise NotImplementedError(f"sql_type {dtype=}")
|
694
|
+
|
695
|
+
|
688
696
|
def write_pandas(
|
689
697
|
conn: FakeSnowflakeConnection,
|
690
698
|
df: pd.DataFrame,
|
@@ -702,7 +710,18 @@ def write_pandas(
|
|
702
710
|
table_type: Literal["", "temp", "temporary", "transient"] = "",
|
703
711
|
**kwargs: Any,
|
704
712
|
) -> WritePandasResult:
|
705
|
-
|
713
|
+
name = table_name
|
714
|
+
if schema:
|
715
|
+
name = f"{schema}.{name}"
|
716
|
+
if database:
|
717
|
+
name = f"{database}.{name}"
|
718
|
+
|
719
|
+
if auto_create_table:
|
720
|
+
cols = [f"{c} {sql_type(t)}" for c, t in df.dtypes.to_dict().items()]
|
721
|
+
|
722
|
+
conn.cursor().execute(f"CREATE TABLE IF NOT EXISTS {name} ({','.join(cols)})")
|
723
|
+
|
724
|
+
count = conn._insert_df(df, name) # noqa: SLF001
|
706
725
|
|
707
726
|
# mocks https://docs.snowflake.com/en/sql-reference/sql/copy-into-table.html#output
|
708
727
|
mock_copy_results = [("fakesnow/file0.txt", "LOADED", count, count, 1, 0, None, None, None, None)]
|
@@ -1122,12 +1122,10 @@ def timestamp_ntz_ns(expression: exp.Expression) -> exp.Expression:
|
|
1122
1122
|
|
1123
1123
|
if (
|
1124
1124
|
isinstance(expression, exp.DataType)
|
1125
|
-
and expression.this == exp.DataType.Type.
|
1125
|
+
and expression.this == exp.DataType.Type.TIMESTAMPNTZ
|
1126
1126
|
and exp.DataTypeParam(this=exp.Literal(this="9", is_string=False)) in expression.expressions
|
1127
1127
|
):
|
1128
|
-
|
1129
|
-
del new.args["expressions"]
|
1130
|
-
return new
|
1128
|
+
return exp.DataType(this=exp.DataType.Type.TIMESTAMP)
|
1131
1129
|
|
1132
1130
|
return expression
|
1133
1131
|
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: fakesnow
|
3
|
-
Version: 0.9.
|
3
|
+
Version: 0.9.13
|
4
4
|
Summary: Fake Snowflake Connector for Python. Run, mock and test Snowflake DB locally.
|
5
5
|
License: Apache License
|
6
6
|
Version 2.0, January 2004
|
@@ -213,7 +213,7 @@ License-File: LICENSE
|
|
213
213
|
Requires-Dist: duckdb~=0.10.0
|
214
214
|
Requires-Dist: pyarrow
|
215
215
|
Requires-Dist: snowflake-connector-python
|
216
|
-
Requires-Dist: sqlglot~=23.
|
216
|
+
Requires-Dist: sqlglot~=23.14.0
|
217
217
|
Provides-Extra: dev
|
218
218
|
Requires-Dist: build~=1.0; extra == "dev"
|
219
219
|
Requires-Dist: pandas-stubs; extra == "dev"
|
@@ -1,7 +1,7 @@
|
|
1
1
|
[project]
|
2
2
|
name = "fakesnow"
|
3
3
|
description = "Fake Snowflake Connector for Python. Run, mock and test Snowflake DB locally."
|
4
|
-
version = "0.9.
|
4
|
+
version = "0.9.13"
|
5
5
|
readme = "README.md"
|
6
6
|
license = { file = "LICENSE" }
|
7
7
|
classifiers = ["License :: OSI Approved :: MIT License"]
|
@@ -11,7 +11,7 @@ dependencies = [
|
|
11
11
|
"duckdb~=0.10.0",
|
12
12
|
"pyarrow",
|
13
13
|
"snowflake-connector-python",
|
14
|
-
"sqlglot~=23.
|
14
|
+
"sqlglot~=23.14.0",
|
15
15
|
]
|
16
16
|
|
17
17
|
[project.urls]
|
@@ -5,9 +5,7 @@ from __future__ import annotations
|
|
5
5
|
import datetime
|
6
6
|
import json
|
7
7
|
import tempfile
|
8
|
-
from collections.abc import Sequence
|
9
8
|
from decimal import Decimal
|
10
|
-
from typing import cast
|
11
9
|
|
12
10
|
import pandas as pd
|
13
11
|
import pytest
|
@@ -19,6 +17,7 @@ from pandas.testing import assert_frame_equal
|
|
19
17
|
from snowflake.connector.cursor import ResultMetadata
|
20
18
|
|
21
19
|
import fakesnow
|
20
|
+
from tests.utils import dindent, indent
|
22
21
|
|
23
22
|
|
24
23
|
def test_alter_table(cur: snowflake.connector.cursor.SnowflakeCursor):
|
@@ -871,6 +870,12 @@ def test_identifier(cur: snowflake.connector.cursor.SnowflakeCursor):
|
|
871
870
|
assert cur.fetchall() == [(1,)]
|
872
871
|
|
873
872
|
|
873
|
+
def test_nop_regexes():
|
874
|
+
with fakesnow.patch(nop_regexes=["^CALL.*"]), snowflake.connector.connect() as conn, conn.cursor() as cur:
|
875
|
+
cur.execute("call this_procedure_does_not_exist('foo', 'bar);")
|
876
|
+
assert cur.fetchall() == [("Statement executed successfully.",)]
|
877
|
+
|
878
|
+
|
874
879
|
def test_non_existent_table_throws_snowflake_exception(cur: snowflake.connector.cursor.SnowflakeCursor):
|
875
880
|
with pytest.raises(snowflake.connector.errors.ProgrammingError) as _:
|
876
881
|
cur.execute("select * from this_table_does_not_exist")
|
@@ -1500,161 +1505,3 @@ def test_json_extract_cast_as_varchar(dcur: snowflake.connector.cursor.DictCurso
|
|
1500
1505
|
|
1501
1506
|
dcur.execute("SELECT j:str::number as c_str_number, j:number::number as c_num_number FROM example")
|
1502
1507
|
assert dcur.fetchall() == [{"C_STR_NUMBER": 100, "C_NUM_NUMBER": 100}]
|
1503
|
-
|
1504
|
-
|
1505
|
-
def test_write_pandas_quoted_column_names(conn: snowflake.connector.SnowflakeConnection):
|
1506
|
-
with conn.cursor(snowflake.connector.cursor.DictCursor) as dcur:
|
1507
|
-
# colunmn names with spaces
|
1508
|
-
dcur.execute('create table customers (id int, "first name" varchar)')
|
1509
|
-
df = pd.DataFrame.from_records(
|
1510
|
-
[
|
1511
|
-
{"ID": 1, "first name": "Jenny"},
|
1512
|
-
{"ID": 2, "first name": "Jasper"},
|
1513
|
-
]
|
1514
|
-
)
|
1515
|
-
snowflake.connector.pandas_tools.write_pandas(conn, df, "CUSTOMERS")
|
1516
|
-
|
1517
|
-
dcur.execute("select * from customers")
|
1518
|
-
|
1519
|
-
assert dcur.fetchall() == [
|
1520
|
-
{"ID": 1, "first name": "Jenny"},
|
1521
|
-
{"ID": 2, "first name": "Jasper"},
|
1522
|
-
]
|
1523
|
-
|
1524
|
-
|
1525
|
-
def test_write_pandas_array(conn: snowflake.connector.SnowflakeConnection):
|
1526
|
-
with conn.cursor() as cur:
|
1527
|
-
cur.execute("create table customers (ID int, FIRST_NAME varchar, LAST_NAME varchar, ORDERS array)")
|
1528
|
-
|
1529
|
-
df = pd.DataFrame.from_records(
|
1530
|
-
[
|
1531
|
-
{"ID": 1, "FIRST_NAME": "Jenny", "LAST_NAME": "P", "ORDERS": ["A", "B"]},
|
1532
|
-
{"ID": 2, "FIRST_NAME": "Jasper", "LAST_NAME": "M", "ORDERS": ["C", "D"]},
|
1533
|
-
]
|
1534
|
-
)
|
1535
|
-
snowflake.connector.pandas_tools.write_pandas(conn, df, "CUSTOMERS")
|
1536
|
-
|
1537
|
-
cur.execute("select * from customers")
|
1538
|
-
|
1539
|
-
assert indent(cur.fetchall()) == [
|
1540
|
-
(1, "Jenny", "P", '[\n "A",\n "B"\n]'),
|
1541
|
-
(2, "Jasper", "M", '[\n "C",\n "D"\n]'),
|
1542
|
-
]
|
1543
|
-
|
1544
|
-
|
1545
|
-
def test_write_pandas_timestamp_ntz(conn: snowflake.connector.SnowflakeConnection):
|
1546
|
-
# compensate for https://github.com/duckdb/duckdb/issues/7980
|
1547
|
-
with conn.cursor() as cur:
|
1548
|
-
cur.execute("create table example (UPDATE_AT_NTZ timestamp_ntz(9))")
|
1549
|
-
# cur.execute("create table example (UPDATE_AT_NTZ timestamp)")
|
1550
|
-
|
1551
|
-
now_utc = datetime.datetime.now(pytz.utc)
|
1552
|
-
df = pd.DataFrame([(now_utc,)], columns=["UPDATE_AT_NTZ"])
|
1553
|
-
snowflake.connector.pandas_tools.write_pandas(conn, df, "EXAMPLE")
|
1554
|
-
|
1555
|
-
cur.execute("select * from example")
|
1556
|
-
|
1557
|
-
assert cur.fetchall() == [(now_utc.replace(tzinfo=None),)]
|
1558
|
-
|
1559
|
-
|
1560
|
-
def test_write_pandas_partial_columns(conn: snowflake.connector.SnowflakeConnection):
|
1561
|
-
with conn.cursor() as cur:
|
1562
|
-
cur.execute("create table customers (ID int, FIRST_NAME varchar, LAST_NAME varchar)")
|
1563
|
-
|
1564
|
-
df = pd.DataFrame.from_records(
|
1565
|
-
[
|
1566
|
-
{"ID": 1, "FIRST_NAME": "Jenny"},
|
1567
|
-
{"ID": 2, "FIRST_NAME": "Jasper"},
|
1568
|
-
]
|
1569
|
-
)
|
1570
|
-
snowflake.connector.pandas_tools.write_pandas(conn, df, "CUSTOMERS")
|
1571
|
-
|
1572
|
-
cur.execute("select id, first_name, last_name from customers")
|
1573
|
-
|
1574
|
-
# columns not in dataframe will receive their default value
|
1575
|
-
assert cur.fetchall() == [(1, "Jenny", None), (2, "Jasper", None)]
|
1576
|
-
|
1577
|
-
|
1578
|
-
def test_write_pandas_dict_as_varchar(conn: snowflake.connector.SnowflakeConnection):
|
1579
|
-
with conn.cursor() as cur:
|
1580
|
-
cur.execute("create or replace table example (vc varchar, o object)")
|
1581
|
-
|
1582
|
-
df = pd.DataFrame([({"kind": "vc", "count": 1}, {"kind": "obj", "amount": 2})], columns=["VC", "O"])
|
1583
|
-
snowflake.connector.pandas_tools.write_pandas(conn, df, "EXAMPLE")
|
1584
|
-
|
1585
|
-
cur.execute("select * from example")
|
1586
|
-
|
1587
|
-
# returned values are valid json strings
|
1588
|
-
# NB: snowflake orders object keys alphabetically, we don't
|
1589
|
-
r = cur.fetchall()
|
1590
|
-
assert [(sort_keys(r[0][0], indent=None), sort_keys(r[0][1], indent=2))] == [
|
1591
|
-
('{"count":1,"kind":"vc"}', '{\n "amount": 2,\n "kind": "obj"\n}')
|
1592
|
-
]
|
1593
|
-
|
1594
|
-
|
1595
|
-
def test_write_pandas_dict_different_keys(conn: snowflake.connector.SnowflakeConnection):
|
1596
|
-
with conn.cursor() as cur:
|
1597
|
-
cur.execute("create or replace table customers (notes variant)")
|
1598
|
-
|
1599
|
-
df = pd.DataFrame.from_records(
|
1600
|
-
[
|
1601
|
-
# rows have dicts with unique keys and values
|
1602
|
-
{"NOTES": {"k": "v1"}},
|
1603
|
-
# test single and double quoting
|
1604
|
-
{"NOTES": {"k2": ["v'2", 'v"3']}},
|
1605
|
-
]
|
1606
|
-
)
|
1607
|
-
snowflake.connector.pandas_tools.write_pandas(conn, df, "CUSTOMERS")
|
1608
|
-
|
1609
|
-
cur.execute("select * from customers")
|
1610
|
-
|
1611
|
-
assert indent(cur.fetchall()) == [('{\n "k": "v1"\n}',), ('{\n "k2": [\n "v\'2",\n "v\\"3"\n ]\n}',)]
|
1612
|
-
|
1613
|
-
|
1614
|
-
def test_write_pandas_db_schema(conn: snowflake.connector.SnowflakeConnection):
|
1615
|
-
with conn.cursor() as cur:
|
1616
|
-
cur.execute("create database db2")
|
1617
|
-
cur.execute("create schema db2.schema2")
|
1618
|
-
cur.execute("create or replace table db2.schema2.customers (ID int, FIRST_NAME varchar, LAST_NAME varchar)")
|
1619
|
-
|
1620
|
-
df = pd.DataFrame.from_records(
|
1621
|
-
[
|
1622
|
-
{"ID": 1, "FIRST_NAME": "Jenny"},
|
1623
|
-
{"ID": 2, "FIRST_NAME": "Jasper"},
|
1624
|
-
]
|
1625
|
-
)
|
1626
|
-
snowflake.connector.pandas_tools.write_pandas(conn, df, "CUSTOMERS", "DB2", "SCHEMA2")
|
1627
|
-
|
1628
|
-
cur.execute("select id, first_name, last_name from db2.schema2.customers")
|
1629
|
-
|
1630
|
-
# columns not in dataframe will receive their default value
|
1631
|
-
assert cur.fetchall() == [(1, "Jenny", None), (2, "Jasper", None)]
|
1632
|
-
|
1633
|
-
|
1634
|
-
def indent(rows: Sequence[tuple] | Sequence[dict]) -> list[tuple]:
|
1635
|
-
# indent duckdb json strings tuple values to match snowflake json strings
|
1636
|
-
assert isinstance(rows[0], tuple)
|
1637
|
-
return [
|
1638
|
-
(*[json.dumps(json.loads(c), indent=2) if (isinstance(c, str) and c.startswith(("[", "{"))) else c for c in r],)
|
1639
|
-
for r in rows
|
1640
|
-
]
|
1641
|
-
|
1642
|
-
|
1643
|
-
def dindent(rows: Sequence[tuple] | Sequence[dict]) -> list[dict]:
|
1644
|
-
# indent duckdb json strings dict values to match snowflake json strings
|
1645
|
-
assert isinstance(rows[0], dict)
|
1646
|
-
return [
|
1647
|
-
{
|
1648
|
-
k: json.dumps(json.loads(v), indent=2) if (isinstance(v, str) and v.startswith(("[", "{"))) else v
|
1649
|
-
for k, v in cast(dict, r).items()
|
1650
|
-
}
|
1651
|
-
for r in rows
|
1652
|
-
]
|
1653
|
-
|
1654
|
-
|
1655
|
-
def sort_keys(sdict: str, indent: int | None = 2) -> str:
|
1656
|
-
return json.dumps(
|
1657
|
-
json.loads(sdict, object_pairs_hook=lambda x: dict(sorted(x))),
|
1658
|
-
indent=indent,
|
1659
|
-
separators=None if indent else (",", ":"),
|
1660
|
-
)
|
@@ -594,6 +594,10 @@ def test_show_schemas() -> None:
|
|
594
594
|
|
595
595
|
def test_tag() -> None:
|
596
596
|
assert sqlglot.parse_one("ALTER TABLE table1 SET TAG foo='bar'", read="snowflake").transform(tag) == SUCCESS_NOP
|
597
|
+
assert (
|
598
|
+
sqlglot.parse_one("ALTER TABLE db1.schema1.table1 SET TAG foo.bar='baz'", read="snowflake").transform(tag)
|
599
|
+
== SUCCESS_NOP
|
600
|
+
)
|
597
601
|
|
598
602
|
|
599
603
|
def test_timestamp_ntz_ns() -> None:
|
@@ -0,0 +1,164 @@
|
|
1
|
+
from __future__ import annotations
|
2
|
+
|
3
|
+
import datetime
|
4
|
+
import json
|
5
|
+
|
6
|
+
import pandas as pd
|
7
|
+
import pytz
|
8
|
+
import snowflake.connector
|
9
|
+
import snowflake.connector.cursor
|
10
|
+
import snowflake.connector.pandas_tools
|
11
|
+
|
12
|
+
from tests.utils import indent
|
13
|
+
|
14
|
+
|
15
|
+
def test_write_pandas_auto_create(conn: snowflake.connector.SnowflakeConnection):
|
16
|
+
with conn.cursor() as cur:
|
17
|
+
df = pd.DataFrame.from_records(
|
18
|
+
[
|
19
|
+
{"ID": 1, "FIRST_NAME": "Jenny"},
|
20
|
+
{"ID": 2, "FIRST_NAME": "Jasper"},
|
21
|
+
]
|
22
|
+
)
|
23
|
+
snowflake.connector.pandas_tools.write_pandas(conn, df, "CUSTOMERS", auto_create_table=True)
|
24
|
+
|
25
|
+
cur.execute("select id, first_name from customers")
|
26
|
+
|
27
|
+
assert cur.fetchall() == [(1, "Jenny"), (2, "Jasper")]
|
28
|
+
|
29
|
+
|
30
|
+
def test_write_pandas_quoted_column_names(conn: snowflake.connector.SnowflakeConnection):
|
31
|
+
with conn.cursor(snowflake.connector.cursor.DictCursor) as dcur:
|
32
|
+
# colunmn names with spaces
|
33
|
+
dcur.execute('create table customers (id int, "first name" varchar)')
|
34
|
+
df = pd.DataFrame.from_records(
|
35
|
+
[
|
36
|
+
{"ID": 1, "first name": "Jenny"},
|
37
|
+
{"ID": 2, "first name": "Jasper"},
|
38
|
+
]
|
39
|
+
)
|
40
|
+
snowflake.connector.pandas_tools.write_pandas(conn, df, "CUSTOMERS")
|
41
|
+
|
42
|
+
dcur.execute("select * from customers")
|
43
|
+
|
44
|
+
assert dcur.fetchall() == [
|
45
|
+
{"ID": 1, "first name": "Jenny"},
|
46
|
+
{"ID": 2, "first name": "Jasper"},
|
47
|
+
]
|
48
|
+
|
49
|
+
|
50
|
+
def test_write_pandas_array(conn: snowflake.connector.SnowflakeConnection):
|
51
|
+
with conn.cursor() as cur:
|
52
|
+
cur.execute("create table customers (ID int, FIRST_NAME varchar, LAST_NAME varchar, ORDERS array)")
|
53
|
+
|
54
|
+
df = pd.DataFrame.from_records(
|
55
|
+
[
|
56
|
+
{"ID": 1, "FIRST_NAME": "Jenny", "LAST_NAME": "P", "ORDERS": ["A", "B"]},
|
57
|
+
{"ID": 2, "FIRST_NAME": "Jasper", "LAST_NAME": "M", "ORDERS": ["C", "D"]},
|
58
|
+
]
|
59
|
+
)
|
60
|
+
snowflake.connector.pandas_tools.write_pandas(conn, df, "CUSTOMERS")
|
61
|
+
|
62
|
+
cur.execute("select * from customers")
|
63
|
+
|
64
|
+
assert indent(cur.fetchall()) == [
|
65
|
+
(1, "Jenny", "P", '[\n "A",\n "B"\n]'),
|
66
|
+
(2, "Jasper", "M", '[\n "C",\n "D"\n]'),
|
67
|
+
]
|
68
|
+
|
69
|
+
|
70
|
+
def test_write_pandas_timestamp_ntz(conn: snowflake.connector.SnowflakeConnection):
|
71
|
+
# compensate for https://github.com/duckdb/duckdb/issues/7980
|
72
|
+
with conn.cursor() as cur:
|
73
|
+
cur.execute("create table example (UPDATE_AT_NTZ timestamp_ntz(9))")
|
74
|
+
# cur.execute("create table example (UPDATE_AT_NTZ timestamp)")
|
75
|
+
|
76
|
+
now_utc = datetime.datetime.now(pytz.utc)
|
77
|
+
df = pd.DataFrame([(now_utc,)], columns=["UPDATE_AT_NTZ"])
|
78
|
+
snowflake.connector.pandas_tools.write_pandas(conn, df, "EXAMPLE")
|
79
|
+
|
80
|
+
cur.execute("select * from example")
|
81
|
+
|
82
|
+
assert cur.fetchall() == [(now_utc.replace(tzinfo=None),)]
|
83
|
+
|
84
|
+
|
85
|
+
def test_write_pandas_partial_columns(conn: snowflake.connector.SnowflakeConnection):
|
86
|
+
with conn.cursor() as cur:
|
87
|
+
cur.execute("create table customers (ID int, FIRST_NAME varchar, LAST_NAME varchar)")
|
88
|
+
|
89
|
+
df = pd.DataFrame.from_records(
|
90
|
+
[
|
91
|
+
{"ID": 1, "FIRST_NAME": "Jenny"},
|
92
|
+
{"ID": 2, "FIRST_NAME": "Jasper"},
|
93
|
+
]
|
94
|
+
)
|
95
|
+
snowflake.connector.pandas_tools.write_pandas(conn, df, "CUSTOMERS")
|
96
|
+
|
97
|
+
cur.execute("select id, first_name, last_name from customers")
|
98
|
+
|
99
|
+
# columns not in dataframe will receive their default value
|
100
|
+
assert cur.fetchall() == [(1, "Jenny", None), (2, "Jasper", None)]
|
101
|
+
|
102
|
+
|
103
|
+
def test_write_pandas_dict_as_varchar(conn: snowflake.connector.SnowflakeConnection):
|
104
|
+
with conn.cursor() as cur:
|
105
|
+
cur.execute("create or replace table example (vc varchar, o object)")
|
106
|
+
|
107
|
+
df = pd.DataFrame([({"kind": "vc", "count": 1}, {"kind": "obj", "amount": 2})], columns=["VC", "O"])
|
108
|
+
snowflake.connector.pandas_tools.write_pandas(conn, df, "EXAMPLE")
|
109
|
+
|
110
|
+
cur.execute("select * from example")
|
111
|
+
|
112
|
+
# returned values are valid json strings
|
113
|
+
# NB: snowflake orders object keys alphabetically, we don't
|
114
|
+
r = cur.fetchall()
|
115
|
+
assert [(sort_keys(r[0][0], indent=None), sort_keys(r[0][1], indent=2))] == [
|
116
|
+
('{"count":1,"kind":"vc"}', '{\n "amount": 2,\n "kind": "obj"\n}')
|
117
|
+
]
|
118
|
+
|
119
|
+
|
120
|
+
def test_write_pandas_dict_different_keys(conn: snowflake.connector.SnowflakeConnection):
|
121
|
+
with conn.cursor() as cur:
|
122
|
+
cur.execute("create or replace table customers (notes variant)")
|
123
|
+
|
124
|
+
df = pd.DataFrame.from_records(
|
125
|
+
[
|
126
|
+
# rows have dicts with unique keys and values
|
127
|
+
{"NOTES": {"k": "v1"}},
|
128
|
+
# test single and double quoting
|
129
|
+
{"NOTES": {"k2": ["v'2", 'v"3']}},
|
130
|
+
]
|
131
|
+
)
|
132
|
+
snowflake.connector.pandas_tools.write_pandas(conn, df, "CUSTOMERS")
|
133
|
+
|
134
|
+
cur.execute("select * from customers")
|
135
|
+
|
136
|
+
assert indent(cur.fetchall()) == [('{\n "k": "v1"\n}',), ('{\n "k2": [\n "v\'2",\n "v\\"3"\n ]\n}',)]
|
137
|
+
|
138
|
+
|
139
|
+
def test_write_pandas_db_schema(conn: snowflake.connector.SnowflakeConnection):
|
140
|
+
with conn.cursor() as cur:
|
141
|
+
cur.execute("create database db2")
|
142
|
+
cur.execute("create schema db2.schema2")
|
143
|
+
cur.execute("create or replace table db2.schema2.customers (ID int, FIRST_NAME varchar, LAST_NAME varchar)")
|
144
|
+
|
145
|
+
df = pd.DataFrame.from_records(
|
146
|
+
[
|
147
|
+
{"ID": 1, "FIRST_NAME": "Jenny"},
|
148
|
+
{"ID": 2, "FIRST_NAME": "Jasper"},
|
149
|
+
]
|
150
|
+
)
|
151
|
+
snowflake.connector.pandas_tools.write_pandas(conn, df, "CUSTOMERS", "DB2", "SCHEMA2")
|
152
|
+
|
153
|
+
cur.execute("select id, first_name, last_name from db2.schema2.customers")
|
154
|
+
|
155
|
+
# columns not in dataframe will receive their default value
|
156
|
+
assert cur.fetchall() == [(1, "Jenny", None), (2, "Jasper", None)]
|
157
|
+
|
158
|
+
|
159
|
+
def sort_keys(sdict: str, indent: int | None = 2) -> str:
|
160
|
+
return json.dumps(
|
161
|
+
json.loads(sdict, object_pairs_hook=lambda x: dict(sorted(x))),
|
162
|
+
indent=indent,
|
163
|
+
separators=None if indent else (",", ":"),
|
164
|
+
)
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|