fakesnow 0.7.1__py3-none-any.whl → 0.8.1__py3-none-any.whl

fakesnow/__main__.py

@@ -0,0 +1,4 @@
+if __name__ == "__main__":
+    import fakesnow.cli
+
+    fakesnow.cli.main()

fakesnow/cli.py

@@ -0,0 +1,28 @@
+import runpy
+import sys
+from collections.abc import Sequence
+
+import fakesnow
+
+USAGE = "Usage: fakesnow <path> | -m <module> [<arg>]..."
+
+
+def main(args: Sequence[str] = sys.argv) -> int:
+    if len(args) < 2 or (len(args) == 2 and args[1] == "-m"):
+        print(USAGE, file=sys.stderr)
+        return 42
+
+    with fakesnow.patch():
+        if args[1] == "-m":
+            module = args[2]
+            sys.argv = args[2:]
+
+            # add current directory to path to mimic python -m
+            sys.path.insert(0, "")
+            runpy.run_module(module, run_name="__main__", alter_sys=True)
+        else:
+            path = args[1]
+            sys.argv = args[1:]
+            runpy.run_path(path, run_name="__main__")
+
+    return 0

fakesnow/fakes.py

@@ -1,9 +1,12 @@
 from __future__ import annotations
 
+import os
 import re
+import sys
 from collections.abc import Iterable, Iterator, Sequence
+from string import Template
 from types import TracebackType
-from typing import TYPE_CHECKING, Any, Literal, Optional, Union, cast
+from typing import TYPE_CHECKING, Any, Literal, Optional, cast
 
 import duckdb
 
@@ -26,6 +29,12 @@ import fakesnow.info_schema as info_schema
 import fakesnow.transforms as transforms
 
 SCHEMA_UNSET = "schema_unset"
+SUCCESS_SQL = "SELECT 'Statement executed successfully.' as status"
+DATABASE_CREATED_SQL = Template("SELECT 'Database ${name} successfully created.' as status")
+TABLE_CREATED_SQL = Template("SELECT 'Table ${name} successfully created.' as status")
+DROPPED_SQL = Template("SELECT '${name} successfully dropped.' as status")
+SCHEMA_CREATED_SQL = Template("SELECT 'Schema ${name} successfully created.' as status")
+INSERTED_SQL = Template("SELECT ${count} as 'number of rows inserted'")
 
 
 class FakeSnowflakeCursor:
@@ -49,6 +58,7 @@ class FakeSnowflakeCursor:
         self._last_sql = None
         self._last_params = None
         self._sqlstate = None
+        self._arraysize = 1
         self._converter = snowflake.connector.converter.SnowflakeConverter()
 
     def __enter__(self) -> Self:
@@ -62,6 +72,19 @@ class FakeSnowflakeCursor:
     ) -> bool:
         return False
 
+    @property
+    def arraysize(self) -> int:
+        return self._arraysize
+
+    @arraysize.setter
+    def arraysize(self, value: int) -> None:
+        self._arraysize = value
+
+    def close(self) -> bool:
+        self._last_sql = None
+        self._last_params = None
+        return True
+
     def describe(self, command: str, *args: Any, **kwargs: Any) -> list[ResultMetadata]:
         """Return the schema of the result without executing the query.
 
@@ -79,8 +102,9 @@ class FakeSnowflakeCursor:
     def description(self) -> list[ResultMetadata]:
         # use a cursor to avoid destroying an unfetched result on the main connection
         with self._duck_conn.cursor() as cur:
-            assert self._conn.database, "Not implemented when database is None"
-            assert self._conn.schema, "Not implemented when schema is None"
+            # TODO: allow sql alchemy connection with no database or schema
+            assert self._conn.database, ".description not implemented when database is None"
+            assert self._conn.schema, ".description not implemented when schema is None"
 
             # match database and schema used on the main connection
             cur.execute(f"SET SCHEMA = '{self._conn.database}.{self._conn.schema}'")
@@ -145,6 +169,8 @@ class FakeSnowflakeCursor:
             .transform(transforms.parse_json)
             # indices_to_json_extract must be before regex_substr
             .transform(transforms.indices_to_json_extract)
+            .transform(transforms.json_extract_as_varchar)
+            .transform(transforms.flatten)
             .transform(transforms.regex_replace)
             .transform(transforms.regex_substr)
             .transform(transforms.values_columns)
@@ -162,7 +188,8 @@ class FakeSnowflakeCursor:
         try:
             self._last_sql = sql
             self._last_params = params
-            # print(f"{sql};")
+            if os.environ.get("FAKESNOW_DEBUG"):
+                print(f"{sql};", file=sys.stderr)
             self._duck_conn.execute(sql, params)
         except duckdb.BinderException as e:
             msg = e.args[0]
@@ -171,6 +198,15 @@ class FakeSnowflakeCursor:
             # minimal processing to make it look like a snowflake exception, message content may differ
             msg = cast(str, e.args[0]).split("\n")[0]
             raise snowflake.connector.errors.ProgrammingError(msg=msg, errno=2003, sqlstate="42S02") from None
+        except duckdb.TransactionException as e:
+            if "cannot rollback - no transaction is active" in str(
+                e
+            ) or "cannot commit - no transaction is active" in str(e):
+                # snowflake doesn't error on rollback or commit outside a tx
+                self._duck_conn.execute(SUCCESS_SQL)
+                self._last_sql = SUCCESS_SQL
+            else:
+                raise e
 
         if cmd == "USE DATABASE" and (ident := expression.find(exp.Identifier)) and isinstance(ident.this, str):
             self._conn.database = ident.this.upper()
@@ -183,6 +219,33 @@ class FakeSnowflakeCursor:
         if create_db_name := transformed.args.get("create_db_name"):
             # we created a new database, so create the info schema extensions
             self._duck_conn.execute(info_schema.creation_sql(create_db_name))
+            created_sql = DATABASE_CREATED_SQL.substitute(name=create_db_name)
+            self._duck_conn.execute(created_sql)
+            self._last_sql = created_sql
+
+        if cmd == "CREATE SCHEMA" and (ident := expression.find(exp.Identifier)) and isinstance(ident.this, str):
+            name = ident.this if ident.quoted else ident.this.upper()
+            created_sql = SCHEMA_CREATED_SQL.substitute(name=name)
+            self._duck_conn.execute(created_sql)
+            self._last_sql = created_sql
+
+        if cmd == "CREATE TABLE" and (ident := expression.find(exp.Identifier)) and isinstance(ident.this, str):
+            name = ident.this if ident.quoted else ident.this.upper()
+            created_sql = TABLE_CREATED_SQL.substitute(name=name)
+            self._duck_conn.execute(created_sql)
+            self._last_sql = created_sql
+
+        if cmd.startswith("DROP") and (ident := expression.find(exp.Identifier)) and isinstance(ident.this, str):
+            name = ident.this if ident.quoted else ident.this.upper()
+            dropped_sql = DROPPED_SQL.substitute(name=name)
+            self._duck_conn.execute(dropped_sql)
+            self._last_sql = dropped_sql
+
+        if cmd == "INSERT":
+            (count,) = self._duck_conn.fetchall()[0]
+            inserted_sql = INSERTED_SQL.substitute(count=count)
+            self._duck_conn.execute(inserted_sql)
+            self._last_sql = inserted_sql
 
         if table_comment := cast(tuple[exp.Table, str], transformed.args.get("table_comment")):
             # record table comment
@@ -232,19 +295,22 @@ class FakeSnowflakeCursor:
         return self._duck_conn.fetch_df()
 
     def fetchone(self) -> dict | tuple | None:
+        result = self.fetchmany(1)
+        return result[0] if result else None
+
+    def fetchmany(self, size: int | None = None) -> list[tuple] | list[dict]:
+        # https://peps.python.org/pep-0249/#fetchmany
+        size = size or self._arraysize
         if not self._use_dict_result:
-            return cast(Union[tuple, None], self._duck_conn.fetchone())
+            return cast(list[tuple], self._duck_conn.fetchmany(size))
 
         if not self._arrow_table:
             self._arrow_table = self._duck_conn.fetch_arrow_table()
-            self._arrow_table_fetch_one_index = -1
+            self._arrow_table_fetch_index = -size
 
-        self._arrow_table_fetch_one_index += 1
+        self._arrow_table_fetch_index += size
 
-        try:
-            return self._arrow_table.take([self._arrow_table_fetch_one_index]).to_pylist()[0]
-        except pyarrow.lib.ArrowIndexError:
-            return None
+        return self._arrow_table.slice(offset=self._arrow_table_fetch_index, length=size).to_pylist()
 
     def get_result_batches(self) -> list[ResultBatch] | None:
         # rows_per_batch is approximate
@@ -322,13 +388,10 @@ class FakeSnowflakeCursor:
                 return ResultMetadata(
                     name=column_name, type_code=12, display_size=None, internal_size=None, precision=0, scale=9, is_nullable=True               # noqa: E501
                 )
-            elif column_type == "JSON[]":
-                return ResultMetadata(
-                    name=column_name, type_code=10, display_size=None, internal_size=None, precision=None, scale=None, is_nullable=True               # noqa: E501
-                )
             elif column_type == "JSON":
+                # TODO: correctly map OBJECT and ARRAY see https://github.com/tekumara/fakesnow/issues/26
                 return ResultMetadata(
-                    name=column_name, type_code=9, display_size=None, internal_size=None, precision=None, scale=None, is_nullable=True               # noqa: E501
+                    name=column_name, type_code=5, display_size=None, internal_size=None, precision=None, scale=None, is_nullable=True               # noqa: E501
                 )
             else:
                 # TODO handle more types

fakesnow/info_schema.py

@@ -1,7 +1,7 @@
-from string import Template
-
 """Info schema extension tables/views used for storing snowflake metadata not captured by duckdb."""
 
+from string import Template
+
 # use ext prefix in columns to disambiguate when joining with information_schema.tables
 SQL_CREATE_INFORMATION_SCHEMA_TABLES_EXT = Template(
     """
@@ -40,8 +40,7 @@ case when starts_with(data_type, 'DECIMAL') or data_type='BIGINT' then 'NUMBER'
      when data_type='DOUBLE' then 'FLOAT'
      when data_type='BLOB' then 'BINARY'
      when data_type='TIMESTAMP' then 'TIMESTAMP_NTZ'
-     when data_type='JSON[]' then 'ARRAY'
-     when data_type='JSON' then 'OBJECT'
+     when data_type='JSON' then 'VARIANT'
      else data_type end as data_type,
 ext_character_maximum_length as character_maximum_length, ext_character_octet_length as character_octet_length,
 case when data_type='BIGINT' then 38

fakesnow/transforms.py

@@ -155,6 +155,49 @@ def extract_text_length(expression: exp.Expression) -> exp.Expression:
     return expression
 
 
+def flatten(expression: exp.Expression) -> exp.Expression:
+    """Flatten an array.
+
+    See https://docs.snowflake.com/en/sql-reference/functions/flatten
+
+    TODO: return index.
+    TODO: support objects.
+    """
+    if (
+        isinstance(expression, exp.Lateral)
+        and isinstance(expression.this, exp.Explode)
+        and (alias := expression.args.get("alias"))
+        # always true; when no explicit alias provided this will be _flattened
+        and isinstance(alias, exp.TableAlias)
+    ):
+        explode_expression = expression.this.this.expression
+
+        return exp.Lateral(
+            this=exp.Unnest(
+                expressions=[
+                    exp.Anonymous(
+                        # duckdb unnests in reserve, so we reverse the list to match
+                        # the order of the original array (and snowflake)
+                        this="list_reverse",
+                        expressions=[
+                            exp.Cast(
+                                this=explode_expression,
+                                to=exp.DataType(
+                                    this=exp.DataType.Type.ARRAY,
+                                    expressions=[exp.DataType(this=exp.DataType.Type.JSON, nested=False, prefix=False)],
+                                    nested=True,
+                                ),
+                            )
+                        ],
+                    )
+                ]
+            ),
+            alias=exp.TableAlias(this=alias.this, columns=[exp.Identifier(this="VALUE", quoted=False)]),
+        )
+
+    return expression
+
+
 def float_to_double(expression: exp.Expression) -> exp.Expression:
     """Convert float to double for 64 bit precision.
 
@@ -176,10 +219,10 @@ def indices_to_json_extract(expression: exp.Expression) -> exp.Expression:
     and object indices, see
     https://docs.snowflake.com/en/sql-reference/data-types-semistructured#accessing-elements-of-an-object-by-key
 
-    Duckdb uses the -> operator, or the json_extract function, see
+    Duckdb uses the -> operator, aka the json_extract function, see
     https://duckdb.org/docs/extensions/json#json-extraction-functions
 
-    This works for Snowflake arrays too because we convert them to JSON[] in duckdb.
+    This works for Snowflake arrays too because we convert them to JSON in duckdb.
     """
     if (
         isinstance(expression, exp.Bracket)
@@ -259,6 +302,20 @@ def integer_precision(expression: exp.Expression) -> exp.Expression:
     return expression
 
 
+def json_extract_as_varchar(expression: exp.Expression) -> exp.Expression:
+    """Return raw unquoted string when casting json extraction to varchar.
+
+    This must run after indices_to_json_extract.
+
+    Duckdb uses the ->> operator, aka the json_extract_string function, see
+    https://duckdb.org/docs/extensions/json#json-extraction-functions
+    """
+    if isinstance(expression, exp.Cast) and (extract := expression.this) and isinstance(extract, exp.JSONExtract):
+        return exp.JSONExtractScalar(this=extract.this, expression=extract.expression)
+
+    return expression
+
+
 def object_construct(expression: exp.Expression) -> exp.Expression:
     """Convert object_construct to return a json string
 
@@ -355,9 +412,13 @@ def regex_substr(expression: exp.Expression) -> exp.Expression:
 
         # which occurrence of the pattern to match
         try:
-            occurrence = expression.args["occurrence"]
+            occurrence = int(expression.args["occurrence"].this)
         except KeyError:
-            occurrence = exp.Literal(this="1", is_string=False)
+            occurrence = 1
+
+        # the duckdb dialect increments bracket (ie: index) expressions by 1 because duckdb is 1-indexed,
+        # so we need to compensate by subtracting 1
+        occurrence = exp.Literal(this=str(occurrence - 1), is_string=False)
 
         try:
             regex_parameters_value = str(expression.args["parameters"].this)
@@ -428,7 +489,7 @@ def set_schema(expression: exp.Expression, current_database: str | None) -> exp.
             name = f"{expression.this.name}.main"
         else:
             # SCHEMA
-            if db := expression.this.args.get("db"):
+            if db := expression.this.args.get("db"):  # noqa: SIM108
                 db_name = db.name
             else:
                 # isn't qualified with a database
@@ -578,15 +639,14 @@ def semi_structured_types(expression: exp.Expression) -> exp.Expression:
         exp.Expression: The transformed expression.
     """
 
-    if isinstance(expression, exp.DataType):
-        if expression.this in [exp.DataType.Type.OBJECT, exp.DataType.Type.VARIANT]:
-            new = expression.copy()
-            new.args["this"] = exp.DataType.Type.JSON
-            return new
-        elif expression.this == exp.DataType.Type.ARRAY:
-            new = expression.copy()
-            new.set("expressions", [exp.DataType(this=exp.DataType.Type.JSON)])
-            return new
+    if isinstance(expression, exp.DataType) and expression.this in [
+        exp.DataType.Type.ARRAY,
+        exp.DataType.Type.OBJECT,
+        exp.DataType.Type.VARIANT,
+    ]:
+        new = expression.copy()
+        new.args["this"] = exp.DataType.Type.JSON
+        return new
 
     return expression
 

{fakesnow-0.7.1.dist-info → fakesnow-0.8.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fakesnow
-Version: 0.7.1
+Version: 0.8.1
 Summary: Fake Snowflake Connector for Python. Run Snowflake DB locally.
 License: MIT License
         
@@ -32,7 +32,7 @@ License-File: LICENSE
 Requires-Dist: duckdb ~=0.9.2
 Requires-Dist: pyarrow
 Requires-Dist: snowflake-connector-python
-Requires-Dist: sqlglot ~=19.5.1
+Requires-Dist: sqlglot ~=20.5.0
 Provides-Extra: dev
 Requires-Dist: black ~=23.9 ; extra == 'dev'
 Requires-Dist: build ~=1.0 ; extra == 'dev'
@@ -63,6 +63,24 @@ pip install fakesnow
 
 ## Usage
 
+Run script.py with fakesnow:
+
+```shell
+fakesnow script.py
+```
+
+Or pytest
+
+```shell
+fakesnow -m pytest
+```
+
+`fakesnow` executes `fakesnow.patch` before running the script or module.
+
+### fakesnow.patch
+
+eg:
+
 ```python
 import fakesnow
 import snowflake.connector
@@ -91,6 +109,8 @@ with fakesnow.patch("mymodule.write_pandas"):
     ...
 ```
 
+### pytest fixtures
+
 pytest [fixtures](fakesnow/fixtures.py) are provided for testing. Example _conftest.py_:
 
 ```python
@@ -146,7 +166,8 @@ For more detail see [tests/test_fakes.py](tests/test_fakes.py)
 
 ## Caveats
 
-- VARCHAR field sizes are not enforced unlike Snowflake which will error with "User character length limit (xxx) exceeded by string" when you try to insert a string longer than the column limit.
+- The order of rows is non deterministic and may not match Snowflake unless ORDER BY is fully specified.
+- VARCHAR field sizes are not enforced. Unlike Snowflake which errors with "User character length limit (xxx) exceeded by string" when an inserted string the exceeds the column limit.
 
 ## Contributing
 

fakesnow-0.8.1.dist-info/RECORD

@@ -0,0 +1,16 @@
+fakesnow/__init__.py,sha256=JXg0HSEHSAOSjEE1ZeA_ogNQ2iFXGEuoibQWk7Pz-WM,3133
+fakesnow/__main__.py,sha256=GDrGyNTvBFuqn_UfDjKs7b3LPtU6gDv1KwosVDrukIM,76
+fakesnow/checks.py,sha256=1qVLR0ZB3z3UPij3Hm8hqlkcNLH2QJnwe8OqkoFCwv8,2356
+fakesnow/cli.py,sha256=KN9kSEXZGNh1orfWsmxFfRiu4fcAqDrgLOh6nTKmhV8,737
+fakesnow/expr.py,sha256=CAxuYIUkwI339DQIBzvFF0F-m1tcVGKEPA5rDTzmH9A,892
+fakesnow/fakes.py,sha256=11uyftf-aQoBPfMcpEWQuMwfEsbDJomof6rKovwFaOk,24491
+fakesnow/fixtures.py,sha256=FfVGhfuIQea0_GQKW8H4ZH0DoxrU2ZnMVJj2eobVbnI,518
+fakesnow/info_schema.py,sha256=SQfNp0i46GmL82NUm9IaeZH78Xp1CBDJsEio_0AYjFQ,4503
+fakesnow/py.typed,sha256=B-DLSjYBi7pkKjwxCSdpVj2J02wgfJr-E7B1wOUyxYU,80
+fakesnow/transforms.py,sha256=4c46iE8Krq7Bx6i78ybFxK0rLtA9m21XEzPCIz65Yqw,25297
+fakesnow-0.8.1.dist-info/LICENSE,sha256=BL6v_VTnU7xdsocviIQJMFr3stX_-uRfTyByo3gRu4M,1071
+fakesnow-0.8.1.dist-info/METADATA,sha256=5c7sxmehggVGW5Y7fGbKPQq-F800vUCFjsNwS4Y95Ow,5740
+fakesnow-0.8.1.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+fakesnow-0.8.1.dist-info/entry_points.txt,sha256=2riAUgu928ZIHawtO8EsfrMEJhi-EH-z_Vq7Q44xKPM,47
+fakesnow-0.8.1.dist-info/top_level.txt,sha256=500evXI1IFX9so82cizGIEMHAb_dJNPaZvd2H9dcKTA,24
+fakesnow-0.8.1.dist-info/RECORD,,

fakesnow-0.8.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+fakesnow = fakesnow.cli:main

{fakesnow-0.7.1.dist-info → fakesnow-0.8.1.dist-info}/top_level.txt

@@ -1,2 +1,3 @@
 dist
 fakesnow
+notebooks

fakesnow-0.7.1.dist-info/RECORD

@@ -1,13 +0,0 @@
-fakesnow/__init__.py,sha256=JXg0HSEHSAOSjEE1ZeA_ogNQ2iFXGEuoibQWk7Pz-WM,3133
-fakesnow/checks.py,sha256=1qVLR0ZB3z3UPij3Hm8hqlkcNLH2QJnwe8OqkoFCwv8,2356
-fakesnow/expr.py,sha256=CAxuYIUkwI339DQIBzvFF0F-m1tcVGKEPA5rDTzmH9A,892
-fakesnow/fakes.py,sha256=SMyZ23KV4qHGkb-RVnWi-5m6IrHGJWETT5RtmRN8er0,21508
-fakesnow/fixtures.py,sha256=FfVGhfuIQea0_GQKW8H4ZH0DoxrU2ZnMVJj2eobVbnI,518
-fakesnow/info_schema.py,sha256=3lkpRI_ByXbA1PwZ3hh4PtB9aLAsRbkI4MM_hTaOce8,4544
-fakesnow/py.typed,sha256=B-DLSjYBi7pkKjwxCSdpVj2J02wgfJr-E7B1wOUyxYU,80
-fakesnow/transforms.py,sha256=koYkt55qGMJhi8gujqrBKO-nePcBMp4zrTk80Uw_JrE,23054
-fakesnow-0.7.1.dist-info/LICENSE,sha256=BL6v_VTnU7xdsocviIQJMFr3stX_-uRfTyByo3gRu4M,1071
-fakesnow-0.7.1.dist-info/METADATA,sha256=5vcf4KPwcNwl7xJ-5fBQmHCxHOatcaTZgFDDPFyUY6E,5418
-fakesnow-0.7.1.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-fakesnow-0.7.1.dist-info/top_level.txt,sha256=x8S-sMmvfgNm2_1w0zlIF5YlDs2hR7eNQdVA6TgmPZE,14
-fakesnow-0.7.1.dist-info/RECORD,,