snowflake-connector-python 5.2.0__cp315-cp315-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- snowflake/connector/__init__.py +148 -0
- snowflake/connector/_common/__init__.py +0 -0
- snowflake/connector/_common/extras.py +134 -0
- snowflake/connector/_common/telemetry.py +221 -0
- snowflake/connector/_common/wif_util.py +92 -0
- snowflake/connector/_core/sf_core_python.cp315-win_amd64.pyd +0 -0
- snowflake/connector/_core/sf_core_python.pyi +79 -0
- snowflake/connector/_internal/__init__.py +6 -0
- snowflake/connector/_internal/_private_key_helper.py +31 -0
- snowflake/connector/_internal/api_client/__init__.py +93 -0
- snowflake/connector/_internal/api_client/bridge.py +41 -0
- snowflake/connector/_internal/api_client/client_api.py +976 -0
- snowflake/connector/_internal/arrow.py +47 -0
- snowflake/connector/_internal/arrow_context.py +240 -0
- snowflake/connector/_internal/arrow_stream_async.py +266 -0
- snowflake/connector/_internal/arrow_stream_iterator.cp311-win_amd64.pyd +0 -0
- snowflake/connector/_internal/arrow_stream_iterator.cp312-win_amd64.pyd +0 -0
- snowflake/connector/_internal/arrow_stream_iterator.cp313-win_amd64.pyd +0 -0
- snowflake/connector/_internal/arrow_stream_iterator.cp314-win_amd64.pyd +0 -0
- snowflake/connector/_internal/arrow_stream_iterator.cp314t-win_amd64.pyd +0 -0
- snowflake/connector/_internal/arrow_stream_iterator.cp315-win_amd64.pyd +0 -0
- snowflake/connector/_internal/arrow_stream_iterator.pyi +42 -0
- snowflake/connector/_internal/arrow_stream_utils.py +171 -0
- snowflake/connector/_internal/authenticator_type.py +53 -0
- snowflake/connector/_internal/backward_compatibility.py +245 -0
- snowflake/connector/_internal/binding_converters.py +741 -0
- snowflake/connector/_internal/config_utils.py +71 -0
- snowflake/connector/_internal/connection/__init__.py +39 -0
- snowflake/connector/_internal/connection/connection.py +616 -0
- snowflake/connector/_internal/connection/connection_types.py +13 -0
- snowflake/connector/_internal/connection/constants.py +25 -0
- snowflake/connector/_internal/connection/decorators.py +31 -0
- snowflake/connector/_internal/connection/freezable_proxy.py +104 -0
- snowflake/connector/_internal/connection/queries.py +10 -0
- snowflake/connector/_internal/connection_config_mixin.py +755 -0
- snowflake/connector/_internal/cursor/__init__.py +25 -0
- snowflake/connector/_internal/cursor/base.py +732 -0
- snowflake/connector/_internal/cursor/cursor_types.py +13 -0
- snowflake/connector/_internal/cursor/decorators.py +56 -0
- snowflake/connector/_internal/cursor/query_result.py +202 -0
- snowflake/connector/_internal/cursor/query_result_waiter.py +98 -0
- snowflake/connector/_internal/cursor/result_metadata.py +255 -0
- snowflake/connector/_internal/cursor/result_set_wrapper.py +66 -0
- snowflake/connector/_internal/decorators.py +539 -0
- snowflake/connector/_internal/error_kinds.py +111 -0
- snowflake/connector/_internal/errorcode.py +35 -0
- snowflake/connector/_internal/errorhandler.py +278 -0
- snowflake/connector/_internal/fips.py +28 -0
- snowflake/connector/_internal/logging/__init__.py +52 -0
- snowflake/connector/_internal/logging/config.py +134 -0
- snowflake/connector/_internal/logging/core_logger.py +218 -0
- snowflake/connector/_internal/logging/native_extension_logger.py +144 -0
- snowflake/connector/_internal/logout_config_mapping.py +187 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/ArrayConverter.cpp +57 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/ArrayConverter.hpp +29 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/ArrowTableConverter.cpp +943 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/ArrowTableConverter.hpp +63 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/BinaryConverter.cpp +18 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/BinaryConverter.hpp +26 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/BooleanConverter.cpp +21 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/BooleanConverter.hpp +23 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/CArrowIterator.cpp +118 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/CArrowIterator.hpp +116 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/CArrowStreamIterator.cpp +323 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/CArrowStreamIterator.hpp +133 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/CArrowStreamTableIterator.cpp +275 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/CArrowStreamTableIterator.hpp +168 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/ConverterUtil.cpp +439 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/ConverterUtil.hpp +30 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/DateConverter.cpp +48 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/DateConverter.hpp +46 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/DecFloatConverter.cpp +76 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/DecFloatConverter.hpp +35 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/DecimalConverter.cpp +90 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/DecimalConverter.hpp +63 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/FixedSizeListConverter.cpp +62 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/FixedSizeListConverter.hpp +28 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/FloatConverter.cpp +29 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/FloatConverter.hpp +35 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/IColumnConverter.hpp +17 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/IntConverter.cpp +23 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/IntConverter.hpp +41 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/IntervalConverter.cpp +65 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/IntervalConverter.hpp +54 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/LICENSE.txt +209 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/MapConverter.cpp +79 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/MapConverter.hpp +30 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/ObjectConverter.cpp +52 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/ObjectConverter.hpp +29 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/Python/Common.cpp +8 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/Python/Common.hpp +117 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/Python/Helpers.cpp +54 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/Python/Helpers.hpp +34 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/SnowflakeType.cpp +33 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/SnowflakeType.hpp +48 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/StringConverter.cpp +18 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/StringConverter.hpp +26 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/TimeConverter.cpp +39 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/TimeConverter.hpp +31 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/TimeStampConverter.cpp +326 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/TimeStampConverter.hpp +138 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/Util/macros.hpp +14 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/Util/time.cpp +65 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/Util/time.hpp +68 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/arrow_stream_iterator.cpp +13278 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/arrow_stream_iterator.pyx +204 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/flatcc/flatcc_accessors.h +109 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/flatcc/flatcc_alloc.h +125 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/flatcc/flatcc_assert.h +45 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/flatcc/flatcc_builder.h +1914 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/flatcc/flatcc_emitter.h +208 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/flatcc/flatcc_endian.h +128 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/flatcc/flatcc_epilogue.h +7 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/flatcc/flatcc_flatbuffers.h +58 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/flatcc/flatcc_identifier.h +144 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/flatcc/flatcc_iov.h +31 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/flatcc/flatcc_prologue.h +8 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/flatcc/flatcc_refmap.h +144 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/flatcc/flatcc_rtconfig.h +160 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/flatcc/flatcc_types.h +98 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/flatcc/flatcc_verifier.h +243 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/flatcc/portable/flatcc_portable.h +14 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/flatcc/portable/paligned_alloc.h +207 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/flatcc/portable/pattributes.h +80 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/flatcc/portable/pdiagnostic.h +83 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/flatcc/portable/pdiagnostic_pop.h +20 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/flatcc/portable/pdiagnostic_push.h +51 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/flatcc/portable/pendian.h +201 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/flatcc/portable/pendian_detect.h +108 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/flatcc/portable/pinline.h +19 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/flatcc/portable/pinttypes.h +52 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/flatcc/portable/portable.h +2 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/flatcc/portable/portable_basic.h +25 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/flatcc/portable/pstatic_assert.h +67 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/flatcc/portable/pstdalign.h +164 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/flatcc/portable/pstdint.h +926 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/flatcc/portable/punaligned.h +182 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/flatcc/portable/pversion.h +6 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/flatcc/portable/pwarnings.h +52 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/flatcc.c +3052 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/nanoarrow.c +3019 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/nanoarrow.h +3354 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/nanoarrow.hpp +344 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/nanoarrow_ipc.c +29305 -0
- snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/nanoarrow_ipc.h +381 -0
- snowflake/connector/_internal/nanoarrow_cpp/Logging/logging.cpp +108 -0
- snowflake/connector/_internal/nanoarrow_cpp/Logging/logging.hpp +47 -0
- snowflake/connector/_internal/nanoarrow_cpp/scripts/.clang-format +8 -0
- snowflake/connector/_internal/nanoarrow_cpp/scripts/format.sh +8 -0
- snowflake/connector/_internal/protobuf_gen/__init__.py +0 -0
- snowflake/connector/_internal/protobuf_gen/database_driver_v1_pb2.py +408 -0
- snowflake/connector/_internal/protobuf_gen/database_driver_v1_pb2.pyi +4487 -0
- snowflake/connector/_internal/protobuf_gen/database_driver_v1_services.py +2088 -0
- snowflake/connector/_internal/protobuf_gen/proto_exception.py +17 -0
- snowflake/connector/_internal/result_batch.py +150 -0
- snowflake/connector/_internal/snowflake_restful.py +222 -0
- snowflake/connector/_internal/snowpark_backcompat_helpers.py +31 -0
- snowflake/connector/_internal/sqlstate.py +11 -0
- snowflake/connector/_internal/statement_utils.py +207 -0
- snowflake/connector/_internal/text_utils.py +112 -0
- snowflake/connector/_internal/type_codes.py +139 -0
- snowflake/connector/_internal/utils.py +65 -0
- snowflake/connector/_internal/write_pandas_operation.py +657 -0
- snowflake/connector/aio/README.md +168 -0
- snowflake/connector/aio/__init__.py +37 -0
- snowflake/connector/aio/connection/__init__.py +17 -0
- snowflake/connector/aio/connection/_connection.py +506 -0
- snowflake/connector/aio/cursor/__init__.py +35 -0
- snowflake/connector/aio/cursor/_base.py +1034 -0
- snowflake/connector/aio/cursor/_dict_cursor.py +78 -0
- snowflake/connector/aio/cursor/_result_set_wrapper.py +35 -0
- snowflake/connector/aio/cursor/_snowflake_cursor.py +72 -0
- snowflake/connector/aio/pandas_tools.py +279 -0
- snowflake/connector/aio/result_batch.py +165 -0
- snowflake/connector/auth/__init__.py +0 -0
- snowflake/connector/auth/workload_identity.py +5 -0
- snowflake/connector/compat.py +22 -0
- snowflake/connector/config_manager.py +392 -0
- snowflake/connector/connection/__init__.py +30 -0
- snowflake/connector/connection/_connection.py +641 -0
- snowflake/connector/connection_config.py +517 -0
- snowflake/connector/constants.py +52 -0
- snowflake/connector/cursor/__init__.py +37 -0
- snowflake/connector/cursor/_base.py +1169 -0
- snowflake/connector/cursor/_chunked_download_reader.py +75 -0
- snowflake/connector/cursor/_dict_cursor.py +78 -0
- snowflake/connector/cursor/_result_set_wrapper.py +35 -0
- snowflake/connector/cursor/_snowflake_cursor.py +72 -0
- snowflake/connector/description.py +7 -0
- snowflake/connector/errors.py +492 -0
- snowflake/connector/network.py +71 -0
- snowflake/connector/pandas_tools.py +191 -0
- snowflake/connector/py.typed +0 -0
- snowflake/connector/result_batch.py +216 -0
- snowflake/connector/telemetry.py +40 -0
- snowflake/connector/time_util.py +11 -0
- snowflake/connector/types.py +134 -0
- snowflake/connector/util_text.py +3 -0
- snowflake/connector/version.py +23 -0
- snowflake_connector_python-5.2.0.dist-info/DELVEWHEEL +2 -0
- snowflake_connector_python-5.2.0.dist-info/METADATA +190 -0
- snowflake_connector_python-5.2.0.dist-info/RECORD +204 -0
- snowflake_connector_python-5.2.0.dist-info/WHEEL +4 -0
- snowflake_connector_python.libs/msvcp140-a4c2229bdc2a2a630acdc095b4d86008.dll +0 -0
|
@@ -0,0 +1,657 @@
|
|
|
1
|
+
"""Write pandas DataFrame to Snowflake table via Parquet stage upload pipeline.
|
|
2
|
+
|
|
3
|
+
This module contains WritePandasConfig (validated parameters) and
|
|
4
|
+
WritePandasOperation (pipeline execution). The public API lives in
|
|
5
|
+
snowflake.connector.pandas_tools.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import re
|
|
11
|
+
import secrets
|
|
12
|
+
import warnings
|
|
13
|
+
|
|
14
|
+
from collections.abc import Callable, Iterator
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from tempfile import TemporaryDirectory
|
|
17
|
+
from typing import TYPE_CHECKING, Any, Literal, NamedTuple, cast
|
|
18
|
+
|
|
19
|
+
from .._common.extras import pandas
|
|
20
|
+
from ..cursor import SnowflakeCursor
|
|
21
|
+
from ..errors import ProgrammingError
|
|
22
|
+
from .logging import get_logger
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
if TYPE_CHECKING:
|
|
26
|
+
from pandas import DataFrame
|
|
27
|
+
|
|
28
|
+
from ..aio.connection import Connection as AsyncConnection
|
|
29
|
+
from ..connection import Connection
|
|
30
|
+
|
|
31
|
+
logger = get_logger(__name__)
|
|
32
|
+
|
|
33
|
+
# Maps user-facing compression name → Snowflake FILE_FORMAT COMPRESSION value.
|
|
34
|
+
# Parquet FILE_FORMAT doesn't accept "gzip" directly; "auto" handles it.
|
|
35
|
+
# https://docs.snowflake.com/en/sql-reference/sql/copy-into-table#type-parquet
|
|
36
|
+
VALID_COMPRESSIONS_MAP: dict[str, str] = {"gzip": "auto", "snappy": "snappy"}
|
|
37
|
+
|
|
38
|
+
VALID_TABLE_TYPES: set[str] = {"", "temp", "temporary", "transient"}
|
|
39
|
+
|
|
40
|
+
ALLOWED_ICEBERG_CONFIGS: set[str] = {
|
|
41
|
+
"EXTERNAL_VOLUME",
|
|
42
|
+
"CATALOG",
|
|
43
|
+
"BASE_LOCATION",
|
|
44
|
+
"CATALOG_SYNC",
|
|
45
|
+
"STORAGE_SERIALIZATION_POLICY",
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
# ---------------------------------------------------------------------------
|
|
50
|
+
# Module-level utilities (stateless, pure functions)
|
|
51
|
+
# ---------------------------------------------------------------------------
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def quote_identifier(name: str) -> str:
|
|
55
|
+
"""Double-quote a SQL identifier, escaping internal double quotes."""
|
|
56
|
+
return '"' + name.replace('"', '""') + '"'
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
_UNQUOTED_IDENTIFIER_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_$]*$")
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def sql_column_identifier(col: str, quote_identifiers: bool) -> str:
|
|
63
|
+
"""Return the SQL token for a column identifier.
|
|
64
|
+
|
|
65
|
+
When quote_identifiers is True, the name is double-quoted with internal
|
|
66
|
+
double quotes escaped. When False, the name must match the unquoted
|
|
67
|
+
identifier grammar (first character a letter or underscore; remaining
|
|
68
|
+
characters letters, digits, underscores, or dollar signs); any other name
|
|
69
|
+
raises ProgrammingError.
|
|
70
|
+
"""
|
|
71
|
+
name = str(col)
|
|
72
|
+
if quote_identifiers:
|
|
73
|
+
return quote_identifier(name)
|
|
74
|
+
if not _UNQUOTED_IDENTIFIER_RE.fullmatch(name):
|
|
75
|
+
raise ProgrammingError(
|
|
76
|
+
msg=f"Column name {name!r} is not a valid unquoted identifier. "
|
|
77
|
+
"Use quote_identifiers=True to allow other names."
|
|
78
|
+
)
|
|
79
|
+
return name
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def qualify_name(
|
|
83
|
+
database: str | None,
|
|
84
|
+
schema: str | None,
|
|
85
|
+
name: str,
|
|
86
|
+
quote_identifiers: bool,
|
|
87
|
+
) -> str:
|
|
88
|
+
"""Build a fully-qualified object name, optionally quoting each part."""
|
|
89
|
+
parts = [p for p in (database, schema, name) if p is not None]
|
|
90
|
+
if quote_identifiers:
|
|
91
|
+
parts = [quote_identifier(p) for p in parts]
|
|
92
|
+
return ".".join(parts)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def escape_path_for_sql(path: str) -> str:
|
|
96
|
+
"""Escape backslashes and single quotes for use inside SQL string literals."""
|
|
97
|
+
return path.replace("\\", "\\\\").replace("'", "\\'")
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def generate_temp_name(prefix: str) -> str:
|
|
101
|
+
"""Generate a random temporary object name."""
|
|
102
|
+
return f"__WRITE_PANDAS_{prefix}_{secrets.token_hex(8)}"
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _sql_bool(value: bool) -> str:
|
|
106
|
+
"""Convert Python bool to SQL TRUE/FALSE literal."""
|
|
107
|
+
return "TRUE" if value else "FALSE"
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _convert_value_to_sql_option(value: str | bool | int | float) -> str:
|
|
111
|
+
"""Convert a Python value to a SQL option literal for iceberg config.
|
|
112
|
+
|
|
113
|
+
String values are always emitted as one escaped single-quoted literal,
|
|
114
|
+
including values that already start and end with quotes. Backslashes are
|
|
115
|
+
doubled before quotes because Snowflake treats backslash as an escape
|
|
116
|
+
inside single-quoted literals.
|
|
117
|
+
"""
|
|
118
|
+
if isinstance(value, str):
|
|
119
|
+
escaped = value.replace("\\", "\\\\").replace("'", "''")
|
|
120
|
+
return f"'{escaped}'"
|
|
121
|
+
return str(value)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
# ---------------------------------------------------------------------------
|
|
125
|
+
# WritePandasResult
|
|
126
|
+
# ---------------------------------------------------------------------------
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
class WritePandasResult(NamedTuple):
|
|
130
|
+
"""Result of a write_pandas invocation. Backward-compatible with plain tuple."""
|
|
131
|
+
|
|
132
|
+
success: bool
|
|
133
|
+
nchunks: int
|
|
134
|
+
nrows: int
|
|
135
|
+
copy_results: list
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
# ---------------------------------------------------------------------------
|
|
139
|
+
# WritePandasConfig
|
|
140
|
+
# ---------------------------------------------------------------------------
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
class WritePandasConfig:
|
|
144
|
+
"""Validated, immutable configuration for a write_pandas invocation.
|
|
145
|
+
|
|
146
|
+
All input parameters are normalized and validated at construction time.
|
|
147
|
+
Raises ProgrammingError or ValueError for invalid inputs.
|
|
148
|
+
"""
|
|
149
|
+
|
|
150
|
+
def __init__(
|
|
151
|
+
self,
|
|
152
|
+
conn: Connection | AsyncConnection,
|
|
153
|
+
df: DataFrame,
|
|
154
|
+
table_name: str,
|
|
155
|
+
*,
|
|
156
|
+
database: str | None = None,
|
|
157
|
+
schema: str | None = None,
|
|
158
|
+
chunk_size: int | None = None,
|
|
159
|
+
compression: Literal["gzip", "snappy"] = "gzip",
|
|
160
|
+
on_error: str = "abort_statement",
|
|
161
|
+
parallel: int = 4,
|
|
162
|
+
quote_identifiers: bool = True,
|
|
163
|
+
infer_schema: bool = False,
|
|
164
|
+
auto_create_table: bool = False,
|
|
165
|
+
overwrite: bool = False,
|
|
166
|
+
table_type: Literal["", "temp", "temporary", "transient"] = "",
|
|
167
|
+
use_logical_type: bool | None = None,
|
|
168
|
+
iceberg_config: dict[str, str] | None = None,
|
|
169
|
+
bulk_upload_chunks: bool = False,
|
|
170
|
+
use_vectorized_scanner: bool = False,
|
|
171
|
+
**parquet_kwargs: Any,
|
|
172
|
+
) -> None:
|
|
173
|
+
self.conn = conn
|
|
174
|
+
self.df = df
|
|
175
|
+
self.table_name = table_name
|
|
176
|
+
self.database = database
|
|
177
|
+
self.schema = schema
|
|
178
|
+
self.chunk_size = chunk_size if chunk_size is not None else len(df)
|
|
179
|
+
self.compression = compression
|
|
180
|
+
self.on_error = on_error
|
|
181
|
+
self.parallel = parallel
|
|
182
|
+
self.quote_identifiers = quote_identifiers
|
|
183
|
+
self.infer_schema = infer_schema
|
|
184
|
+
self.auto_create_table = auto_create_table
|
|
185
|
+
self.overwrite = overwrite
|
|
186
|
+
self.table_type = table_type
|
|
187
|
+
self.use_logical_type = use_logical_type
|
|
188
|
+
self.iceberg_config = iceberg_config
|
|
189
|
+
self.bulk_upload = bulk_upload_chunks
|
|
190
|
+
self.use_vectorized_scanner = use_vectorized_scanner
|
|
191
|
+
self.parquet_kwargs = parquet_kwargs
|
|
192
|
+
|
|
193
|
+
self._validate()
|
|
194
|
+
|
|
195
|
+
# -- Validation ---------------------------------------------------------
|
|
196
|
+
|
|
197
|
+
def _validate(self) -> None:
|
|
198
|
+
if self.database is not None and self.schema is None:
|
|
199
|
+
raise ProgrammingError(msg="schema must be provided when database is specified")
|
|
200
|
+
if self.compression not in VALID_COMPRESSIONS_MAP:
|
|
201
|
+
raise ProgrammingError(
|
|
202
|
+
msg=f"Invalid compression: '{self.compression}'. "
|
|
203
|
+
f"Supported values: {', '.join(sorted(VALID_COMPRESSIONS_MAP))}"
|
|
204
|
+
)
|
|
205
|
+
if self.table_type.lower() not in VALID_TABLE_TYPES:
|
|
206
|
+
raise ValueError(
|
|
207
|
+
f"Invalid table_type: '{self.table_type}'. "
|
|
208
|
+
f"Supported values: "
|
|
209
|
+
f"{', '.join(repr(t) for t in sorted(VALID_TABLE_TYPES))}"
|
|
210
|
+
)
|
|
211
|
+
if self.iceberg_config:
|
|
212
|
+
normalized = {k.upper() for k in self.iceberg_config}
|
|
213
|
+
invalid = normalized - ALLOWED_ICEBERG_CONFIGS
|
|
214
|
+
if invalid:
|
|
215
|
+
raise ProgrammingError(
|
|
216
|
+
msg=f"Invalid iceberg configurations option(s) provided {', '.join(sorted(invalid))}"
|
|
217
|
+
)
|
|
218
|
+
if not self.quote_identifiers:
|
|
219
|
+
for col in self.df.columns:
|
|
220
|
+
sql_column_identifier(str(col), False)
|
|
221
|
+
|
|
222
|
+
# -- Branching properties -----------------------------------------------
|
|
223
|
+
|
|
224
|
+
@property
|
|
225
|
+
def needs_inference(self) -> bool:
|
|
226
|
+
return self.auto_create_table or self.overwrite or self.infer_schema
|
|
227
|
+
|
|
228
|
+
@property
|
|
229
|
+
def needs_table_creation(self) -> bool:
|
|
230
|
+
return self.auto_create_table or self.overwrite
|
|
231
|
+
|
|
232
|
+
@property
|
|
233
|
+
def needs_truncate(self) -> bool:
|
|
234
|
+
return self.overwrite and not self.auto_create_table
|
|
235
|
+
|
|
236
|
+
@property
|
|
237
|
+
def needs_swap(self) -> bool:
|
|
238
|
+
return self.overwrite and self.auto_create_table
|
|
239
|
+
|
|
240
|
+
@property
|
|
241
|
+
def binary_as_text_false_on_stage(self) -> bool:
|
|
242
|
+
return self.auto_create_table or self.overwrite
|
|
243
|
+
|
|
244
|
+
@property
|
|
245
|
+
def binary_as_text_false_on_copy(self) -> bool:
|
|
246
|
+
return self.auto_create_table or self.overwrite or self.infer_schema
|
|
247
|
+
|
|
248
|
+
# -- DataFrame inspection -----------------------------------------------
|
|
249
|
+
|
|
250
|
+
def has_tz_aware_columns(self) -> bool:
|
|
251
|
+
return any(hasattr(dtype, "tz") and dtype.tz is not None for dtype in self.df.dtypes)
|
|
252
|
+
|
|
253
|
+
def is_standard_range_index(self) -> bool:
|
|
254
|
+
idx = self.df.index
|
|
255
|
+
return isinstance(idx, pandas.RangeIndex) and idx.start == 0 and idx.step == 1
|
|
256
|
+
|
|
257
|
+
# -- Helpers ------------------------------------------------------------
|
|
258
|
+
|
|
259
|
+
def qualify(self, name: str) -> str:
|
|
260
|
+
"""Build a fully-qualified name using this config's database/schema."""
|
|
261
|
+
return qualify_name(self.database, self.schema, name, self.quote_identifiers)
|
|
262
|
+
|
|
263
|
+
def emit_warnings(self) -> None:
|
|
264
|
+
"""Emit user-facing warnings about the input data.
|
|
265
|
+
|
|
266
|
+
Called from execute() rather than __init__ so that stacklevel
|
|
267
|
+
correctly points to the caller of write_pandas().
|
|
268
|
+
"""
|
|
269
|
+
if self.use_logical_type is not True and self.has_tz_aware_columns():
|
|
270
|
+
warnings.warn(
|
|
271
|
+
"DataFrame contains a datetime column with timezone "
|
|
272
|
+
"information. This data may not round-trip correctly unless "
|
|
273
|
+
"use_logical_type=True is set.",
|
|
274
|
+
UserWarning,
|
|
275
|
+
stacklevel=4,
|
|
276
|
+
)
|
|
277
|
+
if not self.is_standard_range_index():
|
|
278
|
+
warnings.warn(
|
|
279
|
+
"The DataFrame has a non-standard index which will not be "
|
|
280
|
+
"written. Consider resetting the index with "
|
|
281
|
+
"df.reset_index().",
|
|
282
|
+
UserWarning,
|
|
283
|
+
stacklevel=4,
|
|
284
|
+
)
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
# ---------------------------------------------------------------------------
|
|
288
|
+
# WritePandasMixin
|
|
289
|
+
# ---------------------------------------------------------------------------
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
class WritePandasMixin:
|
|
293
|
+
"""Pure SQL-building methods shared by sync and async write_pandas pipelines.
|
|
294
|
+
|
|
295
|
+
Each SQL-builder returns a dict ready to unpack as ``**kwargs`` into
|
|
296
|
+
``cursor.execute()`` — keys ``operation``, ``parameters``, and
|
|
297
|
+
``_force_qmark_paramstyle`` match the canonical cursor API.
|
|
298
|
+
|
|
299
|
+
Subclasses must provide ``self._cfg: WritePandasConfig``.
|
|
300
|
+
"""
|
|
301
|
+
|
|
302
|
+
_cfg: WritePandasConfig
|
|
303
|
+
|
|
304
|
+
def _build_create_stage_sql(self, name: str) -> dict[str, Any]:
|
|
305
|
+
cfg = self._cfg
|
|
306
|
+
mapped = VALID_COMPRESSIONS_MAP[cfg.compression]
|
|
307
|
+
fmt_opts = [f"TYPE=PARQUET COMPRESSION={mapped}"]
|
|
308
|
+
if cfg.binary_as_text_false_on_stage:
|
|
309
|
+
fmt_opts.append("BINARY_AS_TEXT=FALSE")
|
|
310
|
+
return {
|
|
311
|
+
"operation": f"CREATE TEMPORARY STAGE IDENTIFIER(?) FILE_FORMAT=({' '.join(fmt_opts)})",
|
|
312
|
+
"parameters": (name,),
|
|
313
|
+
"_force_qmark_paramstyle": True,
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
def _build_create_file_format_sql(self, name: str) -> dict[str, Any]:
|
|
317
|
+
cfg = self._cfg
|
|
318
|
+
mapped = VALID_COMPRESSIONS_MAP[cfg.compression]
|
|
319
|
+
parts = [f"CREATE TEMPORARY FILE FORMAT IDENTIFIER(?) TYPE=PARQUET COMPRESSION={mapped}"]
|
|
320
|
+
if cfg.use_logical_type is not None:
|
|
321
|
+
parts.append(f"USE_LOGICAL_TYPE={_sql_bool(cfg.use_logical_type)}")
|
|
322
|
+
return {
|
|
323
|
+
"operation": " ".join(parts),
|
|
324
|
+
"parameters": (name,),
|
|
325
|
+
"_force_qmark_paramstyle": True,
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
def _build_copy_into_sql(
|
|
329
|
+
self,
|
|
330
|
+
stage_location: str,
|
|
331
|
+
target_location: str,
|
|
332
|
+
column_type_map: dict[str, str] | None,
|
|
333
|
+
) -> dict[str, Any]:
|
|
334
|
+
cfg = self._cfg
|
|
335
|
+
target_cols: list[str] = []
|
|
336
|
+
select_exprs: list[str] = []
|
|
337
|
+
|
|
338
|
+
for col in cfg.df.columns:
|
|
339
|
+
col_name = sql_column_identifier(col, cfg.quote_identifiers)
|
|
340
|
+
target_cols.append(col_name)
|
|
341
|
+
|
|
342
|
+
# `$1:"field"` path syntax does not support IDENTIFIER(?) or ? bindings.
|
|
343
|
+
# quote_identifier() escapes the name; the path stays quoted even when quote_identifiers is False.
|
|
344
|
+
parquet_ref = f"$1:{quote_identifier(col)}"
|
|
345
|
+
if column_type_map and col.upper() in column_type_map:
|
|
346
|
+
parquet_ref += f"::{column_type_map[col.upper()]}"
|
|
347
|
+
select_exprs.append(f"{parquet_ref} AS {col_name}")
|
|
348
|
+
|
|
349
|
+
# COPY INTO FROM @stage does not support IDENTIFIER(?) bindings for the stage reference;
|
|
350
|
+
# stage_location is a connector-internal name produced by generate_temp_name().
|
|
351
|
+
escaped_stage = stage_location.replace("'", "\\'")
|
|
352
|
+
compression_mapped = VALID_COMPRESSIONS_MAP[cfg.compression]
|
|
353
|
+
|
|
354
|
+
file_format_parts = [f"TYPE=PARQUET COMPRESSION={compression_mapped}"]
|
|
355
|
+
if cfg.binary_as_text_false_on_copy:
|
|
356
|
+
file_format_parts.append("BINARY_AS_TEXT=FALSE")
|
|
357
|
+
if cfg.use_logical_type is not None:
|
|
358
|
+
file_format_parts.append(f"USE_LOGICAL_TYPE={_sql_bool(cfg.use_logical_type)}")
|
|
359
|
+
if cfg.use_vectorized_scanner:
|
|
360
|
+
file_format_parts.append(f"USE_VECTORIZED_SCANNER={_sql_bool(cfg.use_vectorized_scanner)}")
|
|
361
|
+
|
|
362
|
+
sql = (
|
|
363
|
+
f"COPY INTO IDENTIFIER(?) ({', '.join(target_cols)}) "
|
|
364
|
+
f"FROM (SELECT {', '.join(select_exprs)} "
|
|
365
|
+
f"FROM '@{escaped_stage}') "
|
|
366
|
+
f"FILE_FORMAT = ({' '.join(file_format_parts)}) "
|
|
367
|
+
f"PURGE=TRUE ON_ERROR=?"
|
|
368
|
+
)
|
|
369
|
+
return {
|
|
370
|
+
"operation": sql,
|
|
371
|
+
"parameters": (target_location, cfg.on_error),
|
|
372
|
+
"_force_qmark_paramstyle": True,
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
def _build_drop_object_sql(self, name: str, object_type: str) -> dict[str, Any]:
|
|
376
|
+
# object_type sits in a SQL keyword position (TABLE/STAGE), which cannot be
|
|
377
|
+
# parameter-bound; every call site passes a hardcoded literal, never user input.
|
|
378
|
+
# The object name itself is bound via IDENTIFIER(?).
|
|
379
|
+
return {
|
|
380
|
+
"operation": f"DROP {object_type} IF EXISTS IDENTIFIER(?)",
|
|
381
|
+
"parameters": (name,),
|
|
382
|
+
"_force_qmark_paramstyle": True,
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
def _build_put_file_sql(self, stage_location: str, path: Path) -> dict[str, Any]:
|
|
386
|
+
uri = escape_path_for_sql(path.as_posix())
|
|
387
|
+
# PUT does not support IDENTIFIER(?) or ? bindings for stage paths or file URIs.
|
|
388
|
+
# stage_location comes from generate_temp_name(); uri is escaped by escape_path_for_sql().
|
|
389
|
+
return {
|
|
390
|
+
"operation": (
|
|
391
|
+
f"PUT 'file://{uri}' @{stage_location} "
|
|
392
|
+
f"PARALLEL={self._cfg.parallel} AUTO_COMPRESS=FALSE "
|
|
393
|
+
f"SOURCE_COMPRESSION=AUTO_DETECT OVERWRITE=TRUE"
|
|
394
|
+
),
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
def _build_put_directory_sql(self, stage_location: str, directory: str) -> dict[str, Any]:
|
|
398
|
+
uri = escape_path_for_sql(Path(directory).as_posix())
|
|
399
|
+
# PUT does not support IDENTIFIER(?) or ? bindings for stage paths or file URIs.
|
|
400
|
+
# stage_location comes from generate_temp_name(); uri is escaped by escape_path_for_sql().
|
|
401
|
+
return {
|
|
402
|
+
"operation": (
|
|
403
|
+
f"PUT 'file://{uri}/*' @{stage_location} "
|
|
404
|
+
f"PARALLEL={self._cfg.parallel} AUTO_COMPRESS=FALSE "
|
|
405
|
+
f"SOURCE_COMPRESSION=AUTO_DETECT OVERWRITE=TRUE"
|
|
406
|
+
),
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
def _build_infer_column_types_sql(self, stage_location: str, file_format_location: str) -> dict[str, Any]:
|
|
410
|
+
return {
|
|
411
|
+
"operation": "SELECT * FROM TABLE(INFER_SCHEMA(LOCATION => ?, FILE_FORMAT => ?))",
|
|
412
|
+
"parameters": (f"@{stage_location}", file_format_location),
|
|
413
|
+
"_force_qmark_paramstyle": True,
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
def _build_create_table_sql(
|
|
417
|
+
self,
|
|
418
|
+
target_location: str,
|
|
419
|
+
column_type_map: dict[str, str] | None,
|
|
420
|
+
) -> dict[str, Any]:
|
|
421
|
+
cfg = self._cfg
|
|
422
|
+
col_defs = []
|
|
423
|
+
for col in cfg.df.columns:
|
|
424
|
+
col_type = column_type_map.get(col.upper(), "VARIANT") if column_type_map else "VARIANT"
|
|
425
|
+
col_name = sql_column_identifier(col, cfg.quote_identifiers)
|
|
426
|
+
col_defs.append(f"{col_name} {col_type}")
|
|
427
|
+
|
|
428
|
+
table_type_clause = cfg.table_type.upper() + " " if cfg.table_type else ""
|
|
429
|
+
iceberg_prefix = "ICEBERG " if cfg.iceberg_config else ""
|
|
430
|
+
iceberg_clause = self._build_iceberg_config_sql() if cfg.iceberg_config else ""
|
|
431
|
+
|
|
432
|
+
return {
|
|
433
|
+
"operation": (
|
|
434
|
+
f"CREATE {table_type_clause}{iceberg_prefix}TABLE IF NOT EXISTS "
|
|
435
|
+
f"IDENTIFIER(?) ({', '.join(col_defs)}) {iceberg_clause}"
|
|
436
|
+
),
|
|
437
|
+
"parameters": (target_location,),
|
|
438
|
+
"_force_qmark_paramstyle": True,
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
def _build_truncate_table_sql(self, target_location: str) -> dict[str, Any]:
|
|
442
|
+
return {
|
|
443
|
+
"operation": "TRUNCATE TABLE IF EXISTS IDENTIFIER(?)",
|
|
444
|
+
"parameters": (target_location,),
|
|
445
|
+
"_force_qmark_paramstyle": True,
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
def _build_rename_table_sql(self, target_location: str) -> dict[str, Any]:
|
|
449
|
+
original = self._cfg.qualify(self._cfg.table_name)
|
|
450
|
+
return {
|
|
451
|
+
"operation": "ALTER TABLE IDENTIFIER(?) RENAME TO IDENTIFIER(?)",
|
|
452
|
+
"parameters": (target_location, original),
|
|
453
|
+
"_force_qmark_paramstyle": True,
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
def _build_iceberg_config_sql(self) -> str:
|
|
457
|
+
cfg = self._cfg
|
|
458
|
+
if not cfg.iceberg_config:
|
|
459
|
+
return ""
|
|
460
|
+
normalized = {
|
|
461
|
+
k.upper(): _convert_value_to_sql_option(v) for k, v in cfg.iceberg_config.items() if v is not None
|
|
462
|
+
}
|
|
463
|
+
return " ".join(f"{k}={v}" for k, v in normalized.items())
|
|
464
|
+
|
|
465
|
+
def _iter_chunks(self) -> Iterator[tuple[int, DataFrame]]:
|
|
466
|
+
"""Yield (index, chunk_df). Empty DataFrame yields (0, empty_df)."""
|
|
467
|
+
cfg = self._cfg
|
|
468
|
+
if len(cfg.df) == 0 or cfg.chunk_size == 0:
|
|
469
|
+
yield 0, cfg.df
|
|
470
|
+
return
|
|
471
|
+
for idx, start in enumerate(range(0, len(cfg.df), cfg.chunk_size)):
|
|
472
|
+
yield idx, cfg.df.iloc[start : start + cfg.chunk_size]
|
|
473
|
+
|
|
474
|
+
def _resolve_target_table(self) -> str:
|
|
475
|
+
cfg = self._cfg
|
|
476
|
+
if cfg.needs_swap:
|
|
477
|
+
return cfg.qualify(generate_temp_name("TABLE"))
|
|
478
|
+
return cfg.qualify(cfg.table_name)
|
|
479
|
+
|
|
480
|
+
|
|
481
|
+
# ---------------------------------------------------------------------------
|
|
482
|
+
# WritePandasOperation
|
|
483
|
+
# ---------------------------------------------------------------------------
|
|
484
|
+
|
|
485
|
+
|
|
486
|
+
class WritePandasOperation(WritePandasMixin):
|
|
487
|
+
"""Executes the write_pandas pipeline from a validated config.
|
|
488
|
+
|
|
489
|
+
Accepts a single WritePandasConfig. execute() runs the full pipeline
|
|
490
|
+
without mutating self — all intermediate state is threaded through
|
|
491
|
+
local variables and return values.
|
|
492
|
+
"""
|
|
493
|
+
|
|
494
|
+
def __init__(self, cfg: WritePandasConfig) -> None:
|
|
495
|
+
self._cfg = cfg
|
|
496
|
+
|
|
497
|
+
# -- Main pipeline ---------------------------------------------------
|
|
498
|
+
|
|
499
|
+
def execute(self) -> WritePandasResult:
|
|
500
|
+
"""Run the full pipeline. Returns a WritePandasResult (named tuple)."""
|
|
501
|
+
cfg = self._cfg
|
|
502
|
+
cfg.emit_warnings()
|
|
503
|
+
|
|
504
|
+
# This pipeline runs on the sync connection. ``cfg.conn`` also allows the async one.
|
|
505
|
+
cursor = cast("Connection", cfg.conn).cursor(SnowflakeCursor)
|
|
506
|
+
target_location: str | None = None
|
|
507
|
+
try:
|
|
508
|
+
stage_location = self._create_stage(cursor)
|
|
509
|
+
nchunks, nrows = self._upload_to_stage(cursor, stage_location)
|
|
510
|
+
|
|
511
|
+
column_type_map: dict[str, str] | None = None
|
|
512
|
+
if cfg.needs_inference:
|
|
513
|
+
file_format_location = self._create_file_format(cursor)
|
|
514
|
+
column_type_map = self._infer_column_types(cursor, stage_location, file_format_location)
|
|
515
|
+
|
|
516
|
+
target_location = self._resolve_target_table()
|
|
517
|
+
|
|
518
|
+
if cfg.needs_table_creation:
|
|
519
|
+
self._create_table(cursor, target_location, column_type_map)
|
|
520
|
+
|
|
521
|
+
if cfg.needs_truncate:
|
|
522
|
+
self._truncate_table(cursor, target_location)
|
|
523
|
+
|
|
524
|
+
copy_results = self._copy_into(cursor, stage_location, target_location, column_type_map)
|
|
525
|
+
|
|
526
|
+
if cfg.needs_swap:
|
|
527
|
+
self._swap_tables(cursor, target_location)
|
|
528
|
+
|
|
529
|
+
except Exception:
|
|
530
|
+
if cfg.needs_swap and target_location:
|
|
531
|
+
try:
|
|
532
|
+
self._drop_object(cursor, target_location, "TABLE")
|
|
533
|
+
except Exception:
|
|
534
|
+
logger.warning(
|
|
535
|
+
"write_pandas failed and could not drop temporary "
|
|
536
|
+
"staging table %s created for overwrite swap — "
|
|
537
|
+
"it may need to be dropped manually",
|
|
538
|
+
target_location,
|
|
539
|
+
exc_info=True,
|
|
540
|
+
)
|
|
541
|
+
raise
|
|
542
|
+
finally:
|
|
543
|
+
cursor.close()
|
|
544
|
+
|
|
545
|
+
success = all(row[1] == "LOADED" for row in copy_results)
|
|
546
|
+
return WritePandasResult(success, nchunks, nrows, copy_results)
|
|
547
|
+
|
|
548
|
+
# -- Object lifetime -------------------------------------------------
|
|
549
|
+
|
|
550
|
+
def _drop_object(self, cursor: SnowflakeCursor, name: str, object_type: str) -> None:
|
|
551
|
+
cursor.execute(**self._build_drop_object_sql(name, object_type))
|
|
552
|
+
|
|
553
|
+
def _create_temp_object(
|
|
554
|
+
self,
|
|
555
|
+
cursor: SnowflakeCursor,
|
|
556
|
+
sql_builder: Callable[[str], dict[str, Any]],
|
|
557
|
+
qualified_name: str,
|
|
558
|
+
bare_name: str,
|
|
559
|
+
) -> str:
|
|
560
|
+
"""Try creating in target schema; fall back to current schema on privilege error."""
|
|
561
|
+
try:
|
|
562
|
+
cursor.execute(**sql_builder(qualified_name))
|
|
563
|
+
return qualified_name
|
|
564
|
+
except ProgrammingError:
|
|
565
|
+
cursor.execute(**sql_builder(bare_name))
|
|
566
|
+
return bare_name
|
|
567
|
+
|
|
568
|
+
# -- Stage & upload --------------------------------------------------
|
|
569
|
+
|
|
570
|
+
def _create_stage(self, cursor: SnowflakeCursor) -> str:
|
|
571
|
+
cfg = self._cfg
|
|
572
|
+
name = generate_temp_name("STAGE")
|
|
573
|
+
qualified = cfg.qualify(name)
|
|
574
|
+
return self._create_temp_object(cursor, self._build_create_stage_sql, qualified, name)
|
|
575
|
+
|
|
576
|
+
def _upload_to_stage(self, cursor: SnowflakeCursor, stage_location: str) -> tuple[int, int]:
|
|
577
|
+
"""Write DataFrame chunks to Parquet and PUT to stage.
|
|
578
|
+
|
|
579
|
+
Returns (nchunks, nrows).
|
|
580
|
+
"""
|
|
581
|
+
cfg = self._cfg
|
|
582
|
+
nchunks = 0
|
|
583
|
+
nrows = 0
|
|
584
|
+
with TemporaryDirectory() as tmp_dir:
|
|
585
|
+
for idx, chunk in self._iter_chunks():
|
|
586
|
+
chunk_path = Path(tmp_dir) / f"file{idx}.txt"
|
|
587
|
+
chunk.to_parquet(path=chunk_path, compression=cfg.compression, **cfg.parquet_kwargs)
|
|
588
|
+
nchunks += 1
|
|
589
|
+
nrows += len(chunk)
|
|
590
|
+
|
|
591
|
+
if not cfg.bulk_upload:
|
|
592
|
+
self._put_file(cursor, stage_location, chunk_path)
|
|
593
|
+
chunk_path.unlink()
|
|
594
|
+
|
|
595
|
+
if cfg.bulk_upload:
|
|
596
|
+
self._put_directory(cursor, stage_location, tmp_dir)
|
|
597
|
+
|
|
598
|
+
return nchunks, nrows
|
|
599
|
+
|
|
600
|
+
def _put_file(self, cursor: SnowflakeCursor, stage_location: str, path: Path) -> None:
|
|
601
|
+
cursor.execute(**self._build_put_file_sql(stage_location, path))
|
|
602
|
+
|
|
603
|
+
def _put_directory(self, cursor: SnowflakeCursor, stage_location: str, directory: str) -> None:
|
|
604
|
+
cursor.execute(**self._build_put_directory_sql(stage_location, directory))
|
|
605
|
+
|
|
606
|
+
# -- Schema inference ------------------------------------------------
|
|
607
|
+
|
|
608
|
+
def _create_file_format(self, cursor: SnowflakeCursor) -> str:
|
|
609
|
+
cfg = self._cfg
|
|
610
|
+
name = generate_temp_name("FILE_FORMAT")
|
|
611
|
+
qualified = cfg.qualify(name)
|
|
612
|
+
return self._create_temp_object(cursor, self._build_create_file_format_sql, qualified, name)
|
|
613
|
+
|
|
614
|
+
def _infer_column_types(
|
|
615
|
+
self,
|
|
616
|
+
cursor: SnowflakeCursor,
|
|
617
|
+
stage_location: str,
|
|
618
|
+
file_format_location: str,
|
|
619
|
+
) -> dict[str, str]:
|
|
620
|
+
"""Run INFER_SCHEMA and return {UPPER_COL_NAME: SQL_TYPE} mapping."""
|
|
621
|
+
rows = cursor.execute(**self._build_infer_column_types_sql(stage_location, file_format_location)).fetchall()
|
|
622
|
+
return {row[0].upper(): row[1] for row in rows}
|
|
623
|
+
|
|
624
|
+
# -- Table management ------------------------------------------------
|
|
625
|
+
|
|
626
|
+
def _create_table(
|
|
627
|
+
self,
|
|
628
|
+
cursor: SnowflakeCursor,
|
|
629
|
+
target_location: str,
|
|
630
|
+
column_type_map: dict[str, str] | None,
|
|
631
|
+
) -> None:
|
|
632
|
+
cursor.execute(**self._build_create_table_sql(target_location, column_type_map))
|
|
633
|
+
|
|
634
|
+
def _truncate_table(self, cursor: SnowflakeCursor, target_location: str) -> None:
|
|
635
|
+
cursor.execute(**self._build_truncate_table_sql(target_location))
|
|
636
|
+
|
|
637
|
+
def _swap_tables(self, cursor: SnowflakeCursor, target_location: str) -> None:
|
|
638
|
+
"""Replace original table with temp target via DROP + RENAME.
|
|
639
|
+
|
|
640
|
+
ALTER TABLE SWAP WITH would be atomic but doesn't work when the
|
|
641
|
+
original table doesn't exist yet or tables are different types
|
|
642
|
+
(TEMPORARY vs permanent).
|
|
643
|
+
"""
|
|
644
|
+
original = self._cfg.qualify(self._cfg.table_name)
|
|
645
|
+
self._drop_object(cursor, original, "TABLE")
|
|
646
|
+
cursor.execute(**self._build_rename_table_sql(target_location))
|
|
647
|
+
|
|
648
|
+
# -- COPY INTO -------------------------------------------------------
|
|
649
|
+
|
|
650
|
+
def _copy_into(
|
|
651
|
+
self,
|
|
652
|
+
cursor: SnowflakeCursor,
|
|
653
|
+
stage_location: str,
|
|
654
|
+
target_location: str,
|
|
655
|
+
column_type_map: dict[str, str] | None,
|
|
656
|
+
) -> list:
|
|
657
|
+
return cursor.execute(**self._build_copy_into_sql(stage_location, target_location, column_type_map)).fetchall()
|