snowflake-connector-python 5.2.0__cp315-cp315-win_amd64.whl

snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/arrow_stream_iterator.pyx

@@ -0,0 +1,204 @@
+# distutils: language = c++
+# cython: language_level=3
+# cython: freethreading_compatible = True
+
+from cpython.ref cimport PyObject
+from libc.stdint cimport int64_t, uintptr_t
+from libcpp.memory cimport unique_ptr
+
+from .errorcode import ER_FAILED_TO_CONVERT_ROW_TO_PYTHON_TYPE
+from ..errors import InterfaceError
+
+try:
+    import pyarrow as pa
+except ImportError:
+    pass
+
+
+# Import ReturnVal from C++ (defined in CArrowIterator.hpp)
+cdef extern from "CArrowIterator.hpp" namespace "sf":
+    cdef cppclass ReturnVal:
+        PyObject* successObj
+        PyObject* exception
+
+
+# Import the C++ stream iterator
+cdef extern from "CArrowStreamIterator.hpp" namespace "sf":
+    cdef cppclass CArrowStreamIterator:
+        @staticmethod
+        unique_ptr[CArrowStreamIterator] from_stream(
+            int64_t stream_ptr,
+            PyObject* context,
+            bint use_numpy,
+            bint use_dict_result
+        )
+        ReturnVal next()
+        object nextN(int64_t size)
+        object nextAll()
+
+
+cdef extern from "CArrowStreamTableIterator.hpp" namespace "sf":
+    cdef cppclass CArrowStreamTableIterator:
+        @staticmethod
+        unique_ptr[CArrowStreamTableIterator] from_stream(
+            int64_t stream_ptr,
+            PyObject* context,
+            bint number_to_decimal,
+            bint force_microsecond_precision,
+        )
+        ReturnVal next()
+        uintptr_t getArrowArrayPtr()
+        uintptr_t getArrowSchemaPtr()
+        uintptr_t getConvertedSchemaPtr()
+
+
+cdef class ArrowStreamIterator:
+    """
+    Python wrapper for C++ Arrow stream iterator.
+
+    Reads directly from an ArrowArrayStream pointer. The C++ implementation
+    uses Py_BEGIN_ALLOW_THREADS/Py_END_ALLOW_THREADS to release the GIL during
+    potentially blocking I/O operations (e.g., fetching data chunks from S3).
+    """
+    cdef unique_ptr[CArrowStreamIterator] iterator
+    cdef object arrow_context
+
+    def __cinit__(
+        self,
+        int64_t stream_ptr,
+        object arrow_context,
+        object use_dict_result=False,
+        object use_numpy=False
+    ):
+        """
+        Initialize the stream iterator.
+
+        Parameters
+        ----------
+        stream_ptr : int
+            Pointer to ArrowArrayStream (as integer)
+        arrow_context : ArrowConverterContext
+            Context object for conversions
+        use_dict_result : bool
+            If True, return dicts instead of tuples
+        use_numpy : bool
+            If True, use numpy types for numeric data
+        """
+        self.arrow_context = arrow_context
+
+        # Create the C++ stream iterator using factory method
+        self.iterator = CArrowStreamIterator.from_stream(
+            stream_ptr,
+            <PyObject*>arrow_context,
+            use_numpy,
+            use_dict_result
+        )
+
+        # Check if creation failed (nullptr returned)
+        if self.iterator.get() == NULL:
+            raise RuntimeError("Failed to initialize stream iterator")
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        """Get next row from stream."""
+        cdef ReturnVal ret
+
+        ret = self.iterator.get().next()
+
+        # Check for exception
+        if ret.exception != NULL:
+            error_msg = <object>ret.exception
+            raise InterfaceError(
+                msg=f'Failed to convert current row, cause: {error_msg}',
+                errno=ER_FAILED_TO_CONVERT_ROW_TO_PYTHON_TYPE,
+            )
+
+        # Check for end of iteration
+        if ret.successObj == NULL:
+            raise StopIteration
+
+        # Return the row
+        row = <object>ret.successObj
+        return row
+
+    def fetch_many(self, int64_t size):
+        """Fetch up to `size` rows as a list in a single C++ call."""
+        try:
+            return self.iterator.get().nextN(size)
+        except Exception as e:
+            raise InterfaceError(
+                msg=f'Failed to convert rows, cause: {e}',
+                errno=ER_FAILED_TO_CONVERT_ROW_TO_PYTHON_TYPE,
+            )
+
+    def fetch_all(self):
+        """Fetch all remaining rows as a list in a single C++ call."""
+        try:
+            return self.iterator.get().nextAll()
+        except Exception as e:
+            raise InterfaceError(
+                msg=f'Failed to convert rows, cause: {e}',
+                errno=ER_FAILED_TO_CONVERT_ROW_TO_PYTHON_TYPE,
+            )
+
+cdef class ArrowStreamTableIterator:
+    """
+    Python iterator that reads batches from an ArrowArrayStream and yields
+    one pyarrow.RecordBatch per batch, with Snowflake type conversions
+    applied via ArrowTableConverter.
+    """
+    cdef unique_ptr[CArrowStreamTableIterator] iterator
+    cdef object arrow_context
+
+    def __cinit__(
+        self,
+        int64_t stream_ptr,
+        object arrow_context,
+        bint number_to_decimal = False,
+        bint force_microsecond_precision = False,
+    ):
+        self.arrow_context = arrow_context
+
+        self.iterator = CArrowStreamTableIterator.from_stream(
+            stream_ptr,
+            <PyObject*>arrow_context,
+            number_to_decimal,
+            force_microsecond_precision,
+        )
+        if self.iterator.get() == NULL:
+            raise RuntimeError("Failed to initialize ArrowStreamTableIterator")
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        cdef ReturnVal ret
+        cdef uintptr_t array_ptr
+        cdef uintptr_t schema_ptr
+
+        ret = self.iterator.get().next()
+
+        if ret.exception != NULL:
+            error_msg = <object>ret.exception
+            raise InterfaceError(
+                msg=f'Failed to convert current row, cause: {error_msg}',
+                errno=ER_FAILED_TO_CONVERT_ROW_TO_PYTHON_TYPE,
+            )
+
+        if ret.successObj == NULL:
+            raise StopIteration
+
+        if <object>ret.successObj is None:
+            raise StopIteration
+
+        array_ptr = self.iterator.get().getArrowArrayPtr()
+        schema_ptr = self.iterator.get().getArrowSchemaPtr()
+
+        return pa.RecordBatch._import_from_c(array_ptr, schema_ptr)
+
+    def get_converted_schema(self):
+        cdef uintptr_t schema_ptr = self.iterator.get().getConvertedSchemaPtr()
+
+        return pa.Schema._import_from_c(schema_ptr)

snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/flatcc/flatcc_accessors.h

@@ -0,0 +1,109 @@
+#ifndef FLATCC_ACCESSORS
+#define FLATCC_ACCESSORS
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef UINT8_MAX
+#include <stdint.h>
+#endif
+
+#define __flatcc_basic_scalar_accessors_impl(N, T, W, E)                               \
+  static inline size_t N##__size(void) { return sizeof(T); }                           \
+  static inline T* N##__ptr_add(T* p, size_t i) { return p + i; }                      \
+  static inline const T* N##__const_ptr_add(const T* p, size_t i) { return p + i; }    \
+  static inline T N##_read_from_pe(const void* p) { return N##_cast_from_pe(*(T*)p); } \
+  static inline T N##_read_to_pe(const void* p) { return N##_cast_to_pe(*(T*)p); }     \
+  static inline T N##_read(const void* p) { return *(T*)p; }                           \
+  static inline void N##_write_from_pe(void* p, T v) { *(T*)p = N##_cast_from_pe(v); } \
+  static inline void N##_write_to_pe(void* p, T v) { *(T*)p = N##_cast_to_pe(v); }     \
+  static inline void N##_write(void* p, T v) { *(T*)p = v; }                           \
+  static inline T N##_read_from_le(const void* p) { return N##_cast_from_le(*(T*)p); } \
+  typedef struct {                                                                     \
+    int is_null;                                                                       \
+    T value;                                                                           \
+  } N##_option_t;
+
+#define __flatcc_define_integer_accessors_impl(N, T, W, E)                        \
+  static inline T N##_cast_from_pe(T v) { return (T)E##W##toh((uint##W##_t)v); }  \
+  static inline T N##_cast_to_pe(T v) { return (T)hto##E##W((uint##W##_t)v); }    \
+  static inline T N##_cast_from_le(T v) { return (T)le##W##toh((uint##W##_t)v); } \
+  static inline T N##_cast_to_le(T v) { return (T)htole##W((uint##W##_t)v); }     \
+  static inline T N##_cast_from_be(T v) { return (T)be##W##toh((uint##W##_t)v); } \
+  static inline T N##_cast_to_be(T v) { return (T)htobe##W((uint##W##_t)v); }     \
+  __flatcc_basic_scalar_accessors_impl(N, T, W, E)
+
+#define __flatcc_define_real_accessors_impl(N, T, W, E) \
+  union __##N##_cast {                                  \
+    T v;                                                \
+    uint##W##_t u;                                      \
+  };                                                    \
+  static inline T N##_cast_from_pe(T v) {               \
+    union __##N##_cast x;                               \
+    x.v = v;                                            \
+    x.u = E##W##toh(x.u);                               \
+    return x.v;                                         \
+  }                                                     \
+  static inline T N##_cast_to_pe(T v) {                 \
+    union __##N##_cast x;                               \
+    x.v = v;                                            \
+    x.u = hto##E##W(x.u);                               \
+    return x.v;                                         \
+  }                                                     \
+  static inline T N##_cast_from_le(T v) {               \
+    union __##N##_cast x;                               \
+    x.v = v;                                            \
+    x.u = le##W##toh(x.u);                              \
+    return x.v;                                         \
+  }                                                     \
+  static inline T N##_cast_to_le(T v) {                 \
+    union __##N##_cast x;                               \
+    x.v = v;                                            \
+    x.u = htole##W(x.u);                                \
+    return x.v;                                         \
+  }                                                     \
+  static inline T N##_cast_from_be(T v) {               \
+    union __##N##_cast x;                               \
+    x.v = v;                                            \
+    x.u = be##W##toh(x.u);                              \
+    return x.v;                                         \
+  }                                                     \
+  static inline T N##_cast_to_be(T v) {                 \
+    union __##N##_cast x;                               \
+    x.v = v;                                            \
+    x.u = htobe##W(x.u);                                \
+    return x.v;                                         \
+  }                                                     \
+  __flatcc_basic_scalar_accessors_impl(N, T, W, E)
+
+#define __flatcc_define_integer_accessors(N, T, W, E) \
+  __flatcc_define_integer_accessors_impl(N, T, W, E)
+
+#define __flatcc_define_real_accessors(N, T, W, E) __flatcc_define_real_accessors_impl(N, T, W, E)
+
+#define __flatcc_define_basic_integer_accessors(NS, TN, T, W, E) \
+  __flatcc_define_integer_accessors(NS##TN, T, W, E)
+
+#define __flatcc_define_basic_real_accessors(NS, TN, T, W, E) \
+  __flatcc_define_real_accessors(NS##TN, T, W, E)
+
+#define __flatcc_define_basic_scalar_accessors(NS, E)                                              \
+  __flatcc_define_basic_integer_accessors(                                                         \
+      NS, char, char, 8, E) __flatcc_define_basic_integer_accessors(NS, uint8, uint8_t, 8, E)      \
+      __flatcc_define_basic_integer_accessors(NS, uint16, uint16_t, 16, E)                         \
+          __flatcc_define_basic_integer_accessors(NS, uint32, uint32_t, 32, E)                     \
+              __flatcc_define_basic_integer_accessors(NS, uint64, uint64_t, 64, E)                 \
+                  __flatcc_define_basic_integer_accessors(NS, int8, int8_t, 8, E)                  \
+                      __flatcc_define_basic_integer_accessors(NS, int16, int16_t, 16, E)           \
+                          __flatcc_define_basic_integer_accessors(NS, int32, int32_t, 32, E)       \
+                              __flatcc_define_basic_integer_accessors(NS, int64, int64_t, 64, E)   \
+                                  __flatcc_define_basic_real_accessors(NS, float, float, 32, E)    \
+                                      __flatcc_define_basic_real_accessors(NS, double, double, 64, \
+                                                                           E)
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* FLATCC_ACCESSORS */

snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/flatcc/flatcc_alloc.h

@@ -0,0 +1,125 @@
+#ifndef FLATCC_ALLOC_H
+#define FLATCC_ALLOC_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * These allocation abstractions are __only__ for runtime libraries.
+ *
+ * The flatcc compiler uses Posix allocation routines regardless
+ * of how this file is configured.
+ *
+ * This header makes it possible to use systems where malloc is not
+ * valid to use. In this case the portable library will not help
+ * because it implements Posix / C11 abstractions.
+ *
+ * Systems like FreeRTOS do not work with Posix memory calls and here it
+ * can be helpful to override runtime allocation primitives.
+ *
+ * In general, it is better to customize the allocator and emitter via
+ * flatcc_builder_custom_init and to avoid using the default emitter
+ * specific high level calls the copy out a buffer that must later be
+ * deallocated. This provides full control of allocation withou the need
+ * for this file.
+ *
+ *
+ * IMPORTANT
+ *
+ * If you override malloc, free, etc., make sure your applications
+ * use the same allocation methods. For example, samples/monster.c
+ * and several test cases are no longer guaranteed to work out of the
+ * box.
+ *
+ * The changes must only affect target runtime compilation including the
+ * the runtime library libflatccrt.
+ *
+ * The host system flatcc compiler and the compiler library libflatcc
+ * should NOT be compiled with non-Posix allocation since the compiler
+ * has a dependency on the runtime library and the wrong free operation
+ * might be callled. The safest way to avoid this problem this is to
+ * compile flatcc with the CMake script and the runtime files with a
+ * dedicated build system for the target system.
+ */
+
+#include <stdlib.h>
+
+#ifndef FLATCC_ALLOC
+#define FLATCC_ALLOC(n) malloc(n)
+#endif
+
+#ifndef FLATCC_FREE
+#define FLATCC_FREE(p) free(p)
+#endif
+
+#ifndef FLATCC_REALLOC
+#define FLATCC_REALLOC(p, n) realloc(p, n)
+#endif
+
+#ifndef FLATCC_CALLOC
+#define FLATCC_CALLOC(nm, n) calloc(nm, n)
+#endif
+
+/*
+ * Implements `aligned_alloc` and `aligned_free`.
+ * Even with C11, this implements non-standard aligned_free needed for portable
+ * aligned_alloc implementations.
+ */
+#ifndef FLATCC_USE_GENERIC_ALIGNED_ALLOC
+
+#ifndef FLATCC_NO_PALIGNED_ALLOC
+#include "flatcc/portable/paligned_alloc.h"
+#else
+#if !defined(__aligned_free_is_defined) || !__aligned_free_is_defined
+#define aligned_free free
+#endif
+#endif
+
+#else /* FLATCC_USE_GENERIC_ALIGNED_ALLOC */
+
+#ifndef FLATCC_ALIGNED_ALLOC
+static inline void* __flatcc_aligned_alloc(size_t alignment, size_t size) {
+  char* raw;
+  void* buf;
+  size_t total_size = (size + alignment - 1 + sizeof(void*));
+
+  if (alignment < sizeof(void*)) {
+    alignment = sizeof(void*);
+  }
+  raw = (char*)(size_t)FLATCC_ALLOC(total_size);
+  buf = raw + alignment - 1 + sizeof(void*);
+  buf = (void*)(((size_t)buf) & ~(alignment - 1));
+  ((void**)buf)[-1] = raw;
+  return buf;
+}
+#define FLATCC_ALIGNED_ALLOC(alignment, size) __flatcc_aligned_alloc(alignment, size)
+#endif /* FLATCC_USE_GENERIC_ALIGNED_ALLOC */
+
+#ifndef FLATCC_ALIGNED_FREE
+static inline void __flatcc_aligned_free(void* p) {
+  char* raw;
+
+  if (!p) return;
+  raw = ((void**)p)[-1];
+
+  FLATCC_FREE(raw);
+}
+#define FLATCC_ALIGNED_FREE(p) __flatcc_aligned_free(p)
+#endif
+
+#endif /* FLATCC_USE_GENERIC_ALIGNED_ALLOC */
+
+#ifndef FLATCC_ALIGNED_ALLOC
+#define FLATCC_ALIGNED_ALLOC(a, n) aligned_alloc(a, n)
+#endif
+
+#ifndef FLATCC_ALIGNED_FREE
+#define FLATCC_ALIGNED_FREE(p) aligned_free(p)
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* FLATCC_ALLOC_H */

snowflake/connector/_internal/nanoarrow_cpp/ArrowIterator/flatcc/flatcc_assert.h

@@ -0,0 +1,45 @@
+#ifndef FLATCC_ASSERT_H
+#define FLATCC_ASSERT_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * This assert abstraction is only used for the flatcc runtime library.
+ * The flatcc compiler uses Posix assert routines regardless of how this
+ * file is configured.
+ *
+ * This header makes it possible to use systems where assert is not
+ * valid to use. Note that `<assert.h>` may remain a dependency for static
+ * assertions.
+ *
+ * `FLATCC_ASSERT` is designed to handle errors which cannot be ignored
+ * and could lead to crash. The portable library may use assertions that
+ * are not affected by this macro.
+ *
+ * `FLATCC_ASSERT` defaults to POSIX assert but can be overrided by a
+ * preprocessor definition.
+ *
+ * Runtime assertions can be entirely disabled by defining
+ * `FLATCC_NO_ASSERT`.
+ */
+
+#ifdef FLATCC_NO_ASSERT
+/* NOTE: This will not affect inclusion of <assert.h> for static assertions. */
+#undef FLATCC_ASSERT
+#define FLATCC_ASSERT(x) ((void)0)
+/* Grisu3 is used for floating point conversion in JSON processing. */
+#define GRISU3_NO_ASSERT
+#endif
+
+#ifndef FLATCC_ASSERT
+#include <assert.h>
+#define FLATCC_ASSERT assert
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* FLATCC_ASSERT_H */