snowflake-connector-python 3.16.0__cp313-cp313-win_amd64.whl

snowflake/connector/gzip_decoder.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python
+from __future__ import annotations
+
+import io
+import subprocess
+import zlib
+from logging import getLogger
+from typing import IO, Generator
+
+CHUNK_SIZE = 16384
+MAGIC_NUMBER = 16  # magic number from .vendored.requests/packages/urllib3/response.py
+
+logger = getLogger(__name__)
+
+
+def decompress_raw_data(raw_data_fd: IO, add_bracket: bool = True) -> bytes:
+    """Decompresses raw data from file like object with zlib.
+
+    Args:
+        raw_data_fd: File descriptor object.
+        add_bracket: Whether, or not to add brackets around the output. (Default value = True)
+
+    Returns:
+        A byte array of the decompressed file.
+    """
+    obj = zlib.decompressobj(MAGIC_NUMBER + zlib.MAX_WBITS)
+    writer = io.BytesIO()
+    if add_bracket:
+        writer.write(b"[")
+    d = raw_data_fd.read(CHUNK_SIZE)
+    while d:
+        writer.write(obj.decompress(d))
+        while obj.unused_data != b"":
+            unused_data = obj.unused_data
+            obj = zlib.decompressobj(MAGIC_NUMBER + zlib.MAX_WBITS)
+            writer.write(obj.decompress(unused_data))
+        d = raw_data_fd.read(CHUNK_SIZE)
+        writer.write(obj.flush())
+    if add_bracket:
+        writer.write(b"]")
+    return writer.getvalue()
+
+
+def decompress_raw_data_by_zcat(raw_data_fd: IO, add_bracket: bool = True) -> bytes:
+    """Experimental: Decompresses raw data from file like object with zcat. Otherwise same as decompress_raw_data.
+
+    Args:
+        raw_data_fd: File descriptor object.
+        add_bracket: Whether, or not to add brackets around the output. (Default value = True)
+
+    Returns:
+        A byte array of the decompressed file.
+    """
+    writer = io.BytesIO()
+    if add_bracket:
+        writer.write(b"[")
+    p = subprocess.Popen(["zcat"], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+    writer.write(p.communicate(input=raw_data_fd.read())[0])
+    if add_bracket:
+        writer.write(b"]")
+    return writer.getvalue()
+
+
+def decompress_raw_data_to_unicode_stream(
+    raw_data_fd: IO,
+) -> Generator[str]:
+    """Decompresses a raw data in file like object and yields a Unicode string.
+
+    Args:
+        raw_data_fd: File descriptor object.
+
+    Yields:
+        A string of the decompressed file in chunks.
+    """
+    obj = zlib.decompressobj(MAGIC_NUMBER + zlib.MAX_WBITS)
+    yield "["
+    d = raw_data_fd.read(CHUNK_SIZE)
+    while d:
+        yield obj.decompress(d).decode("utf-8")
+        while obj.unused_data != b"":
+            unused_data = obj.unused_data
+            obj = zlib.decompressobj(MAGIC_NUMBER + zlib.MAX_WBITS)
+            yield obj.decompress(unused_data).decode("utf-8")
+        d = raw_data_fd.read(CHUNK_SIZE)
+    yield obj.flush().decode("utf-8") + "]"

snowflake/connector/local_storage_client.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python
+from __future__ import annotations
+
+import os
+import shutil
+from logging import getLogger
+from typing import TYPE_CHECKING, Any
+
+from .constants import FileHeader, ResultStatus
+from .storage_client import SnowflakeStorageClient
+from .vendored import requests
+
+if TYPE_CHECKING:  # pragma: no cover
+    from .file_transfer_agent import SnowflakeFileMeta
+
+logger = getLogger(__name__)
+
+
+class SnowflakeLocalStorageClient(SnowflakeStorageClient):
+    def __init__(
+        self,
+        meta: SnowflakeFileMeta,
+        stage_info: dict[str, Any],
+        chunk_size: int,
+        unsafe_file_write: bool = False,
+    ) -> None:
+        super().__init__(
+            meta, stage_info, chunk_size, unsafe_file_write=unsafe_file_write
+        )
+        self.data_file = meta.src_file_name
+        self.full_dst_file_name: str = os.path.join(
+            stage_info["location"], os.path.basename(meta.dst_file_name)
+        )
+        if meta.local_location:
+            src_file_name = self.data_file
+            if src_file_name.startswith("/"):
+                src_file_name = src_file_name[1:]
+            self.stage_file_name: str = os.path.join(
+                stage_info["location"], src_file_name
+            )
+            self.full_dst_file_name = os.path.join(
+                meta.local_location, os.path.basename(meta.dst_file_name)
+            )
+
+    def get_file_header(self, filename: str) -> FileHeader | None:
+        """
+        Notes:
+            Checks whether the file exits in specified directory, does not return FileHeader
+        """
+        if os.path.isfile(filename):
+            return FileHeader(None, os.stat(filename).st_size, None)
+        return None
+
+    def download_chunk(self, chunk_id: int) -> None:
+        with open(self.stage_file_name, "rb") as sfd:
+            with open(
+                os.path.join(
+                    self.meta.local_location,
+                    os.path.basename(self.intermediate_dst_path),
+                ),
+                "rb+",
+            ) as tfd:
+                if self.num_of_chunks == 1:
+                    tfd.write(sfd.read())
+                else:
+                    tfd.seek(chunk_id * self.chunk_size)
+                    sfd.seek(chunk_id * self.chunk_size)
+                    tfd.write(sfd.read(self.chunk_size))
+
+    def finish_download(self) -> None:
+        shutil.move(self.intermediate_dst_path, self.full_dst_file_name)
+        self.meta.dst_file_size = os.stat(self.full_dst_file_name).st_size
+        self.meta.result_status = ResultStatus.DOWNLOADED
+
+    def _has_expired_token(self, response: requests.Response) -> bool:
+        return False
+
+    def prepare_upload(self) -> None:
+        super().prepare_upload()
+        with open(self.full_dst_file_name, "wb+") as fd:
+            fd.truncate(self.meta.upload_size)
+
+    def _upload_chunk(self, chunk_id: int, chunk: bytes) -> None:
+        with open(self.full_dst_file_name, "rb+") as tfd:
+            tfd.seek(chunk_id * self.chunk_size)
+            tfd.write(chunk)
+
+    def finish_upload(self) -> None:
+        self.meta.result_status = ResultStatus.UPLOADED
+        self.meta.dst_file_size = self.meta.upload_size

snowflake/connector/log_configuration.py

@@ -0,0 +1,60 @@
+from __future__ import annotations
+
+import logging
+import os
+from logging.handlers import TimedRotatingFileHandler
+
+from snowflake.connector.config_manager import CONFIG_MANAGER
+from snowflake.connector.constants import DIRS
+from snowflake.connector.secret_detector import SecretDetector
+
+LOG_FILE_NAME = "python-connector.log"
+
+
+class EasyLoggingConfigPython:
+    def __init__(self):
+        self.path: str | None = None
+        self.level: str | None = None
+        self.save_logs: bool = False
+        self.parse_config_file()
+
+    def parse_config_file(self):
+        CONFIG_MANAGER.read_config()
+        data = CONFIG_MANAGER.conf_file_cache
+        if log := data.get("log"):
+            self.save_logs = log.get("save_logs", False)
+            self.level = log.get("level", "INFO")
+            self.path = log.get("path", os.path.join(DIRS.user_config_path, "logs"))
+
+            if not os.path.isabs(self.path):
+                raise FileNotFoundError(
+                    f"Log path must be an absolute file path: {self.path}"
+                )
+            # if log path does not exist, create it, else check accessibility
+            if not os.path.exists(self.path):
+                os.makedirs(self.path, exist_ok=True)
+            elif not os.access(self.path, os.R_OK | os.W_OK):
+                raise PermissionError(
+                    f"log path: {self.path} is not accessible, please verify your config file"
+                )
+
+    # create_log() is called outside __init__() so that it can be easily turned off
+    def create_log(self):
+        if self.save_logs:
+            logging.basicConfig(
+                filename=os.path.join(self.path, LOG_FILE_NAME),
+                level=logging.getLevelName(self.level),
+            )
+            for logger_name in ["snowflake.connector", "botocore", "boto3"]:
+                logger = logging.getLogger(logger_name)
+                logger.setLevel(logging.getLevelName(self.level))
+                ch = TimedRotatingFileHandler(
+                    os.path.join(self.path, LOG_FILE_NAME), when="midnight"
+                )
+                ch.setLevel(logging.getLevelName(self.level))
+                ch.setFormatter(
+                    SecretDetector(
+                        "%(asctime)s - %(threadName)s %(filename)s:%(lineno)d - %(funcName)s() - %(levelname)s - %(message)s"
+                    )
+                )
+                logger.addHandler(ch)

snowflake/connector/logging_utils/filters.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+import logging
+
+from snowflake.connector.secret_detector import SecretDetector
+
+
+def add_filter_to_logger_and_children(
+    base_logger_name: str, filter_instance: logging.Filter
+) -> None:
+    # Ensure the base logger exists and apply filter
+    base_logger = logging.getLogger(base_logger_name)
+    if filter_instance not in base_logger.filters:
+        base_logger.addFilter(filter_instance)
+
+    all_loggers_pairs = logging.root.manager.loggerDict.items()
+    for name, obj in all_loggers_pairs:
+        if not name.startswith(base_logger_name + "."):
+            continue
+
+        if not isinstance(obj, logging.Logger):
+            continue  # Skip placeholders
+
+        if filter_instance not in obj.filters:
+            obj.addFilter(filter_instance)
+
+
+class SecretMaskingFilter(logging.Filter):
+    """
+    A logging filter that masks sensitive information in log messages using the SecretDetector utility.
+
+    This filter is designed for scenarios where you want to avoid applying SecretDetector globally
+    as a formatter on all logging handlers. Global masking can introduce unnecessary computational
+    overhead, particularly for internal logs where secrets are already handled explicitly.
+    It would be also easy to bypass unintentionally by simply adding a neighbouring handler to a logger
+    - without SecretDetector set as a formatter.
+
+    On the other hand, libraries or submodules often do not have any handler attached, so formatting can't be
+    configured on those level, while attaching new handler for that can cause unintended log output or its duplication.
+
+    ⚠ Important:
+        - Logging filters do **not** propagate down the logger hierarchy.
+          To apply this filter across a hierarchy, use the `add_filter_to_logger_and_children` utility.
+        - This filter causes **early formatting** of the log message (`record.getMessage()`),
+          meaning `record.args` are merged into `record.msg` prematurely.
+          If you rely on `record.args`, ensure this is the **last** filter in the chain.
+
+    Notes:
+        - The filter directly modifies `record.msg` with the masked version of the message.
+        - It clears `record.args` to prevent re-formatting and ensure safe message output.
+
+    Example:
+        logger.addFilter(SecretMaskingFilter())
+        handler.addFilter(SecretMaskingFilter())
+    """
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        try:
+            # Format the message as it would be
+            message = record.getMessage()
+
+            # Run masking on the whole message
+            masked_data = SecretDetector.mask_secrets(message)
+            record.msg = masked_data.masked_text
+        except Exception as ex:
+            record.msg = SecretDetector.create_formatting_error_log(
+                record, "EXCEPTION - " + str(ex)
+            )
+        finally:
+            record.args = ()  # Avoid format re-application of formatting
+
+        return True  # allow all logs through

snowflake/connector/nanoarrow_cpp/ArrowIterator/ArrayConverter.cpp

@@ -0,0 +1,60 @@
+#include "ArrayConverter.hpp"
+
+#include <memory>
+
+#include "CArrowChunkIterator.hpp"
+#include "CArrowIterator.hpp"
+#include "SnowflakeType.hpp"
+
+namespace sf {
+Logger* ArrayConverter::logger =
+    new Logger("snowflake.connector.ArrayConverter");
+
+void ArrayConverter::generateError(const std::string& msg) const {
+  logger->error(__FILE__, __func__, __LINE__, msg.c_str());
+  PyErr_SetString(PyExc_Exception, msg.c_str());
+}
+
+ArrayConverter::ArrayConverter(ArrowSchemaView* schemaView,
+                               ArrowArrayView* array, PyObject* context,
+                               bool useNumpy) {
+  m_array = array;
+
+  if (schemaView->schema->n_children != 1) {
+    std::string errorInfo = Logger::formatString(
+        "[Snowflake Exception] invalid arrow schema for array items expected 1 "
+        "schema child, but got %d",
+        schemaView->schema->n_children);
+    this->generateError(errorInfo);
+    return;
+  }
+
+  ArrowSchema* item_schema = schemaView->schema->children[0];
+  ArrowArrayView* item_array = array->children[0];
+  m_item_converter = getConverterFromSchema(item_schema, item_array, context,
+                                            useNumpy, logger);
+}
+
+PyObject* ArrayConverter::toPyObject(int64_t rowIndex) const {
+  if (ArrowArrayViewIsNull(m_array, rowIndex)) {
+    Py_RETURN_NONE;
+  }
+
+  // Array item offsets are stored in the second array buffers
+  // Infer start an end of this rows slice by looking at the
+  // current and next offset. If there isn't another offset use
+  // the end of the array instead.
+  int start = m_array->buffer_views[1].data.as_int32[rowIndex];
+  int end = m_array->children[0]->length;
+  if (rowIndex + 1 < m_array->length) {
+    end = m_array->buffer_views[1].data.as_int32[rowIndex + 1];
+  }
+
+  PyObject* list = PyList_New(end - start);
+  for (int i = start; i < end; i++) {
+    PyList_SetItem(list, i - start, m_item_converter->toPyObject(i));
+  }
+  return list;
+}
+
+}  // namespace sf

snowflake/connector/nanoarrow_cpp/ArrowIterator/ArrayConverter.hpp

@@ -0,0 +1,29 @@
+#ifndef PC_ARRAYCONVERTER_HPP
+#define PC_ARRAYCONVERTER_HPP
+
+#include <memory>
+
+#include "IColumnConverter.hpp"
+#include "logging.hpp"
+#include "nanoarrow.h"
+#include "nanoarrow.hpp"
+
+namespace sf {
+
+class ArrayConverter : public IColumnConverter {
+ public:
+  explicit ArrayConverter(ArrowSchemaView* schemaView, ArrowArrayView* array,
+                          PyObject* context, bool useNumpy);
+
+  PyObject* toPyObject(int64_t rowIndex) const override;
+
+ private:
+  void generateError(const std::string& msg) const;
+
+  ArrowArrayView* m_array;
+  std::shared_ptr<sf::IColumnConverter> m_item_converter;
+  static Logger* logger;
+};
+
+}  // namespace sf
+#endif  // PC_ARRAYCONVERTER_HPP

snowflake/connector/nanoarrow_cpp/ArrowIterator/BinaryConverter.cpp

@@ -0,0 +1,19 @@
+#include "BinaryConverter.hpp"
+
+#include <memory>
+
+namespace sf {
+Logger* BinaryConverter::logger =
+    new Logger("snowflake.connector.BinaryConverter");
+
+BinaryConverter::BinaryConverter(ArrowArrayView* array) : m_array(array) {}
+
+PyObject* BinaryConverter::toPyObject(int64_t rowIndex) const {
+  if (ArrowArrayViewIsNull(m_array, rowIndex)) {
+    Py_RETURN_NONE;
+  }
+  ArrowStringView stringView = ArrowArrayViewGetStringUnsafe(m_array, rowIndex);
+  return PyByteArray_FromStringAndSize(stringView.data, stringView.size_bytes);
+}
+
+}  // namespace sf

snowflake/connector/nanoarrow_cpp/ArrowIterator/BinaryConverter.hpp

@@ -0,0 +1,26 @@
+#ifndef PC_BINARYCONVERTER_HPP
+#define PC_BINARYCONVERTER_HPP
+
+#include <memory>
+
+#include "IColumnConverter.hpp"
+#include "logging.hpp"
+#include "nanoarrow.h"
+
+namespace sf {
+
+class BinaryConverter : public IColumnConverter {
+ public:
+  explicit BinaryConverter(ArrowArrayView* array);
+
+  PyObject* toPyObject(int64_t rowIndex) const override;
+
+ private:
+  ArrowArrayView* m_array;
+
+  static Logger* logger;
+};
+
+}  // namespace sf
+
+#endif  // PC_BINARYCONVERTER_HPP

snowflake/connector/nanoarrow_cpp/ArrowIterator/BooleanConverter.cpp

@@ -0,0 +1,21 @@
+#include "BooleanConverter.hpp"
+
+#include <memory>
+
+namespace sf {
+
+BooleanConverter::BooleanConverter(ArrowArrayView* array) : m_array(array) {}
+
+PyObject* BooleanConverter::toPyObject(int64_t rowIndex) const {
+  if (ArrowArrayViewIsNull(m_array, rowIndex)) {
+    Py_RETURN_NONE;
+  }
+
+  if (ArrowArrayViewGetIntUnsafe(m_array, rowIndex)) {
+    Py_RETURN_TRUE;
+  } else {
+    Py_RETURN_FALSE;
+  }
+}
+
+}  // namespace sf

snowflake/connector/nanoarrow_cpp/ArrowIterator/BooleanConverter.hpp

@@ -0,0 +1,23 @@
+#ifndef PC_BOOLEANCONVERTER_HPP
+#define PC_BOOLEANCONVERTER_HPP
+
+#include <memory>
+
+#include "IColumnConverter.hpp"
+#include "nanoarrow.h"
+
+namespace sf {
+
+class BooleanConverter : public IColumnConverter {
+ public:
+  explicit BooleanConverter(ArrowArrayView* array);
+
+  PyObject* toPyObject(int64_t rowIndex) const override;
+
+ private:
+  ArrowArrayView* m_array;
+};
+
+}  // namespace sf
+
+#endif  // PC_BOOLEANCONVERTER_HPP