fakesnow 0.9.22__py3-none-any.whl → 0.9.24__py3-none-any.whl

fakesnow/pandas_tools.py

@@ -0,0 +1,108 @@
+from __future__ import annotations
+
+import json
+from collections.abc import Sequence
+from typing import TYPE_CHECKING, Any, Literal, Optional
+
+import numpy as np
+from duckdb import DuckDBPyConnection
+
+from fakesnow.conn import FakeSnowflakeConnection
+
+if TYPE_CHECKING:
+    # don't require pandas at import time
+    import pandas as pd
+
+
+CopyResult = tuple[
+    str,
+    str,
+    int,
+    int,
+    int,
+    int,
+    Optional[str],
+    Optional[int],
+    Optional[int],
+    Optional[str],
+]
+
+WritePandasResult = tuple[
+    bool,
+    int,
+    int,
+    Sequence[CopyResult],
+]
+
+
+def sql_type(dtype: np.dtype) -> str:
+    if str(dtype) == "int64":
+        return "NUMBER"
+    elif str(dtype) == "object":
+        return "VARCHAR"
+    else:
+        raise NotImplementedError(f"sql_type {dtype=}")
+
+
+def write_pandas(
+    conn: FakeSnowflakeConnection,
+    df: pd.DataFrame,
+    table_name: str,
+    database: str | None = None,
+    schema: str | None = None,
+    chunk_size: int | None = None,
+    compression: str = "gzip",
+    on_error: str = "abort_statement",
+    parallel: int = 4,
+    quote_identifiers: bool = True,
+    auto_create_table: bool = False,
+    create_temp_table: bool = False,
+    overwrite: bool = False,
+    table_type: Literal["", "temp", "temporary", "transient"] = "",
+    **kwargs: Any,
+) -> WritePandasResult:
+    name = table_name
+    if schema:
+        name = f"{schema}.{name}"
+    if database:
+        name = f"{database}.{name}"
+
+    if auto_create_table:
+        cols = [f"{c} {sql_type(t)}" for c, t in df.dtypes.to_dict().items()]
+
+        conn.cursor().execute(f"CREATE TABLE IF NOT EXISTS {name} ({','.join(cols)})")
+
+    count = _insert_df(conn._duck_conn, df, name)  # noqa: SLF001
+
+    # mocks https://docs.snowflake.com/en/sql-reference/sql/copy-into-table.html#output
+    mock_copy_results = [("fakesnow/file0.txt", "LOADED", count, count, 1, 0, None, None, None, None)]
+
+    # return success
+    return (True, len(mock_copy_results), count, mock_copy_results)
+
+
+def _insert_df(duck_conn: DuckDBPyConnection, df: pd.DataFrame, table_name: str) -> int:
+    # Objects in dataframes are written as parquet structs, and snowflake loads parquet structs as json strings.
+    # Whereas duckdb analyses a dataframe see https://duckdb.org/docs/api/python/data_ingestion.html#pandas-dataframes--object-columns
+    # and converts a object to the most specific type possible, eg: dict -> STRUCT, MAP or varchar, and list -> LIST
+    # For dicts see https://github.com/duckdb/duckdb/pull/3985 and https://github.com/duckdb/duckdb/issues/9510
+    #
+    # When the rows have dicts with different keys there isn't a single STRUCT that can cover them, so the type is
+    # varchar and value a string containing a struct representation. In order to support dicts with different keys
+    # we first convert the dicts to json strings. A pity we can't do something inside duckdb and avoid the dataframe
+    # copy and transform in python.
+
+    df = df.copy()
+
+    # Identify columns of type object
+    object_cols = df.select_dtypes(include=["object"]).columns
+
+    # Apply json.dumps to these columns
+    for col in object_cols:
+        # don't jsonify string
+        df[col] = df[col].apply(lambda x: json.dumps(x) if isinstance(x, (dict, list)) else x)
+
+    escaped_cols = ",".join(f'"{col}"' for col in df.columns.to_list())
+    duck_conn.execute(f"INSERT INTO {table_name}({escaped_cols}) SELECT * FROM df")
+
+    return duck_conn.fetchall()[0][0]

fakesnow/server.py

@@ -12,7 +12,7 @@ from starlette.requests import Request
 from starlette.responses import JSONResponse
 from starlette.routing import Route
 
-from fakesnow.arrow import to_ipc
+from fakesnow.arrow import to_ipc, to_rowtype, with_sf_metadata
 from fakesnow.fakes import FakeSnowflakeConnection
 from fakesnow.instance import FakeSnow
 
@@ -52,19 +52,13 @@ async def query_request(request: Request) -> JSONResponse:
         batch_bytes = to_ipc(cur._arrow_table)  # noqa: SLF001
         rowset_b64 = b64encode(batch_bytes).decode("utf-8")
 
+        # TODO: avoid calling with_sf_metadata twice
+        rowtype = to_rowtype(with_sf_metadata(cur._arrow_table.schema))  # noqa: SLF001
+
         return JSONResponse(
             {
                 "data": {
-                    "rowtype": [
-                        {
-                            "name": "'HELLO WORLD'",
-                            "nullable": False,
-                            "type": "text",
-                            "length": 11,
-                            "scale": None,
-                            "precision": None,
-                        }
-                    ],
+                    "rowtype": rowtype,
                     "rowsetBase64": rowset_b64,
                     "total": 1,
                     "queryResultFormat": "arrow",

fakesnow/types.py

@@ -0,0 +1,89 @@
+import re
+from typing import Optional, TypedDict
+
+from snowflake.connector.cursor import ResultMetadata
+
+
+class ColumnInfo(TypedDict):
+    name: str
+    database: str
+    schema: str
+    table: str
+    nullable: bool
+    type: str
+    byteLength: Optional[int]
+    length: Optional[int]
+    scale: Optional[int]
+    precision: Optional[int]
+    collation: Optional[str]
+
+
+duckdb_to_sf_type = {
+    "BIGINT": "fixed",
+    "BLOB": "binary",
+    "BOOLEAN": "boolean",
+    "DATE": "date",
+    "DECIMAL": "fixed",
+    "DOUBLE": "real",
+    "INTEGER": "fixed",
+    "JSON": "variant",
+    "TIME": "time",
+    "TIMESTAMP WITH TIME ZONE": "timestamp_tz",
+    "TIMESTAMP_NS": "timestamp_ntz",
+    "TIMESTAMP": "timestamp_ntz",
+    "VARCHAR": "text",
+}
+
+
+def describe_as_rowtype(describe_results: list) -> list[ColumnInfo]:
+    """Convert duckdb column type to snowflake rowtype returned by the API."""
+
+    def as_column_info(column_name: str, column_type: str) -> ColumnInfo:
+        if not (sf_type := duckdb_to_sf_type.get("DECIMAL" if column_type.startswith("DECIMAL") else column_type)):
+            raise NotImplementedError(f"for column type {column_type}")
+
+        info: ColumnInfo = {
+            "name": column_name,
+            # TODO
+            "database": "",
+            "schema": "",
+            "table": "",
+            # TODO
+            "nullable": True,
+            "type": sf_type,
+            "byteLength": None,
+            "length": None,
+            "scale": None,
+            "precision": None,
+            "collation": None,
+        }
+
+        if column_type.startswith("DECIMAL"):
+            match = re.search(r"\((\d+),(\d+)\)", column_type)
+            info["precision"] = int(match[1]) if match else 38
+            info["scale"] = int(match[2]) if match else 0
+        elif sf_type == "fixed":
+            info["precision"] = 38
+            info["scale"] = 0
+        elif sf_type == "text":
+            # TODO: fetch actual varchar size
+            info["byteLength"] = 16777216
+            info["length"] = 16777216
+        elif sf_type.startswith("time"):
+            info["precision"] = 0
+            info["scale"] = 9
+        elif sf_type == "binary":
+            info["byteLength"] = 8388608
+            info["length"] = 8388608
+
+        return info
+
+    column_infos = [
+        as_column_info(column_name, column_type)
+        for (column_name, column_type, _null, _key, _default, _extra) in describe_results
+    ]
+    return column_infos
+
+
+def describe_as_result_metadata(describe_results: list) -> list[ResultMetadata]:
+    return [ResultMetadata.from_column(c) for c in describe_as_rowtype(describe_results)]  # pyright: ignore[reportArgumentType]

{fakesnow-0.9.22.dist-info → fakesnow-0.9.24.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fakesnow
-Version: 0.9.22
+Version: 0.9.24
 Summary: Fake Snowflake Connector for Python. Run, mock and test Snowflake DB locally.
 License: Apache License
                                    Version 2.0, January 2004
@@ -216,6 +216,7 @@ Requires-Dist: snowflake-connector-python
 Requires-Dist: sqlglot~=25.9.0
 Provides-Extra: dev
 Requires-Dist: build~=1.0; extra == "dev"
+Requires-Dist: dirty-equals; extra == "dev"
 Requires-Dist: pandas-stubs; extra == "dev"
 Requires-Dist: snowflake-connector-python[pandas,secure-local-storage]; extra == "dev"
 Requires-Dist: pre-commit~=3.4; extra == "dev"

{fakesnow-0.9.22.dist-info → fakesnow-0.9.24.dist-info}/RECORD

@@ -1,21 +1,25 @@
 fakesnow/__init__.py,sha256=9tFJJKvowKNW3vfnlmza6hOLN1I52DwChgNc5Ew6CcA,3499
 fakesnow/__main__.py,sha256=GDrGyNTvBFuqn_UfDjKs7b3LPtU6gDv1KwosVDrukIM,76
-fakesnow/arrow.py,sha256=1ypCsf-r2Ven6CuSm-bTLoeq1G31kBD6JnaLvDxpwhU,1218
+fakesnow/arrow.py,sha256=WLkr1nEiNxUcPdzadKSM33sRAiQJsN6LvuzTVIsi3D0,2766
 fakesnow/checks.py,sha256=-QMvdcrRbhN60rnzxLBJ0IkUBWyLR8gGGKKmCS0w9mA,2383
 fakesnow/cli.py,sha256=9qfI-Ssr6mo8UmIlXkUAOz2z2YPBgDsrEVaZv9FjGFs,2201
+fakesnow/conn.py,sha256=Gy_Z7BZRm5yMjV3x6hR4iegDQFdG9aJBjqWdc3iWYFU,5353
+fakesnow/cursor.py,sha256=2PtW9hzfXs3mzv6BBxXLoS-pPtD4otrfQ2KnPNNanGI,19441
 fakesnow/expr.py,sha256=CAxuYIUkwI339DQIBzvFF0F-m1tcVGKEPA5rDTzmH9A,892
-fakesnow/fakes.py,sha256=8roPAjUiVxSZDhxnpsP85sueSa3abZhyoDwM8awZZBY,31376
+fakesnow/fakes.py,sha256=JQTiUkkwPeQrJ8FDWhPFPK6pGwd_aR2oiOrNzCWznlM,187
 fakesnow/fixtures.py,sha256=G-NkVeruSQAJ7fvSS2fR2oysUn0Yra1pohHlOvacKEk,455
 fakesnow/info_schema.py,sha256=DObVOrhzppAFHsdtj4YI9oRISn9SkJUG6ONjVleQQ_Y,6303
 fakesnow/instance.py,sha256=3cJvPRuFy19dMKXbtBLl6imzO48pEw8uTYhZyFDuwhk,3133
 fakesnow/macros.py,sha256=pX1YJDnQOkFJSHYUjQ6ErEkYIKvFI6Ncz_au0vv1csA,265
+fakesnow/pandas_tools.py,sha256=WjyjTV8QUCQQaCGboaEOvx2uo4BkknpWYjtLwkeCY6U,3468
 fakesnow/py.typed,sha256=B-DLSjYBi7pkKjwxCSdpVj2J02wgfJr-E7B1wOUyxYU,80
-fakesnow/server.py,sha256=cTuMzbYL3etm61wZJ7bcnWpcSNoCSTk31gAnl0Kxi20,3183
+fakesnow/server.py,sha256=8dzaLUUXPzCMm6-ESn0CBws6XSwwOpnUuHQAZJ-4SwU,3011
 fakesnow/transforms.py,sha256=ellcY5OBc7mqgL9ChNolrqcCLWXF9RH21Jt88FcFl-I,54419
+fakesnow/types.py,sha256=9Tt83Z7ctc9_v6SYyayXYz4MEI4RZo4zq_uqdj4g3Dk,2681
 fakesnow/variables.py,sha256=WXyPnkeNwD08gy52yF66CVe2twiYC50tztNfgXV4q1k,3032
-fakesnow-0.9.22.dist-info/LICENSE,sha256=kW-7NWIyaRMQiDpryfSmF2DObDZHGR1cJZ39s6B1Svg,11344
-fakesnow-0.9.22.dist-info/METADATA,sha256=Uu-JhX3mgGrgAP3jgLCP3b8YjRhWTh4qXwx-izVDZPM,18020
-fakesnow-0.9.22.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-fakesnow-0.9.22.dist-info/entry_points.txt,sha256=2riAUgu928ZIHawtO8EsfrMEJhi-EH-z_Vq7Q44xKPM,47
-fakesnow-0.9.22.dist-info/top_level.txt,sha256=500evXI1IFX9so82cizGIEMHAb_dJNPaZvd2H9dcKTA,24
-fakesnow-0.9.22.dist-info/RECORD,,
+fakesnow-0.9.24.dist-info/LICENSE,sha256=kW-7NWIyaRMQiDpryfSmF2DObDZHGR1cJZ39s6B1Svg,11344
+fakesnow-0.9.24.dist-info/METADATA,sha256=LHKc6JYn9sxxFh6_i7kqlWz1fmloFv2CCmpalwPVFrE,18064
+fakesnow-0.9.24.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+fakesnow-0.9.24.dist-info/entry_points.txt,sha256=2riAUgu928ZIHawtO8EsfrMEJhi-EH-z_Vq7Q44xKPM,47
+fakesnow-0.9.24.dist-info/top_level.txt,sha256=500evXI1IFX9so82cizGIEMHAb_dJNPaZvd2H9dcKTA,24
+fakesnow-0.9.24.dist-info/RECORD,,