snowflake-connector-python 3.16.0__cp313-cp313-win_amd64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (246) hide show
  1. snowflake/connector/__init__.py +96 -0
  2. snowflake/connector/_query_context_cache.py +276 -0
  3. snowflake/connector/_sql_util.py +48 -0
  4. snowflake/connector/_utils.py +70 -0
  5. snowflake/connector/arrow_context.py +203 -0
  6. snowflake/connector/auth/__init__.py +53 -0
  7. snowflake/connector/auth/_auth.py +586 -0
  8. snowflake/connector/auth/_http_server.py +220 -0
  9. snowflake/connector/auth/_oauth_base.py +400 -0
  10. snowflake/connector/auth/by_plugin.py +219 -0
  11. snowflake/connector/auth/default.py +36 -0
  12. snowflake/connector/auth/idtoken.py +72 -0
  13. snowflake/connector/auth/keypair.py +222 -0
  14. snowflake/connector/auth/no_auth.py +39 -0
  15. snowflake/connector/auth/oauth.py +49 -0
  16. snowflake/connector/auth/oauth_code.py +479 -0
  17. snowflake/connector/auth/oauth_credentials.py +66 -0
  18. snowflake/connector/auth/okta.py +335 -0
  19. snowflake/connector/auth/pat.py +39 -0
  20. snowflake/connector/auth/usrpwdmfa.py +69 -0
  21. snowflake/connector/auth/webbrowser.py +502 -0
  22. snowflake/connector/auth/workload_identity.py +94 -0
  23. snowflake/connector/azure_storage_client.py +276 -0
  24. snowflake/connector/backoff_policies.py +141 -0
  25. snowflake/connector/bind_upload_agent.py +89 -0
  26. snowflake/connector/cache.py +696 -0
  27. snowflake/connector/compat.py +127 -0
  28. snowflake/connector/config_manager.py +496 -0
  29. snowflake/connector/connection.py +2273 -0
  30. snowflake/connector/connection_diagnostic.py +776 -0
  31. snowflake/connector/constants.py +442 -0
  32. snowflake/connector/converter.py +785 -0
  33. snowflake/connector/converter_issue23517.py +87 -0
  34. snowflake/connector/converter_null.py +14 -0
  35. snowflake/connector/converter_snowsql.py +205 -0
  36. snowflake/connector/cursor.py +1951 -0
  37. snowflake/connector/dbapi.py +53 -0
  38. snowflake/connector/description.py +19 -0
  39. snowflake/connector/direct_file_operation_utils.py +88 -0
  40. snowflake/connector/encryption_util.py +220 -0
  41. snowflake/connector/errorcode.py +91 -0
  42. snowflake/connector/errors.py +615 -0
  43. snowflake/connector/externals_utils/__init__.py +0 -0
  44. snowflake/connector/externals_utils/externals_setup.py +27 -0
  45. snowflake/connector/feature.py +4 -0
  46. snowflake/connector/file_compression_type.py +118 -0
  47. snowflake/connector/file_lock.py +72 -0
  48. snowflake/connector/file_transfer_agent.py +1215 -0
  49. snowflake/connector/file_util.py +153 -0
  50. snowflake/connector/gcs_storage_client.py +474 -0
  51. snowflake/connector/gzip_decoder.py +85 -0
  52. snowflake/connector/local_storage_client.py +90 -0
  53. snowflake/connector/log_configuration.py +60 -0
  54. snowflake/connector/logging_utils/__init__.py +0 -0
  55. snowflake/connector/logging_utils/filters.py +72 -0
  56. snowflake/connector/nanoarrow_arrow_iterator.cp313-win_amd64.pyd +0 -0
  57. snowflake/connector/nanoarrow_cpp/ArrowIterator/ArrayConverter.cpp +60 -0
  58. snowflake/connector/nanoarrow_cpp/ArrowIterator/ArrayConverter.hpp +29 -0
  59. snowflake/connector/nanoarrow_cpp/ArrowIterator/BinaryConverter.cpp +19 -0
  60. snowflake/connector/nanoarrow_cpp/ArrowIterator/BinaryConverter.hpp +26 -0
  61. snowflake/connector/nanoarrow_cpp/ArrowIterator/BooleanConverter.cpp +21 -0
  62. snowflake/connector/nanoarrow_cpp/ArrowIterator/BooleanConverter.hpp +23 -0
  63. snowflake/connector/nanoarrow_cpp/ArrowIterator/CArrowChunkIterator.cpp +557 -0
  64. snowflake/connector/nanoarrow_cpp/ArrowIterator/CArrowChunkIterator.hpp +98 -0
  65. snowflake/connector/nanoarrow_cpp/ArrowIterator/CArrowIterator.cpp +125 -0
  66. snowflake/connector/nanoarrow_cpp/ArrowIterator/CArrowIterator.hpp +115 -0
  67. snowflake/connector/nanoarrow_cpp/ArrowIterator/CArrowTableIterator.cpp +1005 -0
  68. snowflake/connector/nanoarrow_cpp/ArrowIterator/CArrowTableIterator.hpp +127 -0
  69. snowflake/connector/nanoarrow_cpp/ArrowIterator/DateConverter.cpp +47 -0
  70. snowflake/connector/nanoarrow_cpp/ArrowIterator/DateConverter.hpp +46 -0
  71. snowflake/connector/nanoarrow_cpp/ArrowIterator/DecFloatConverter.cpp +83 -0
  72. snowflake/connector/nanoarrow_cpp/ArrowIterator/DecFloatConverter.hpp +35 -0
  73. snowflake/connector/nanoarrow_cpp/ArrowIterator/DecimalConverter.cpp +97 -0
  74. snowflake/connector/nanoarrow_cpp/ArrowIterator/DecimalConverter.hpp +72 -0
  75. snowflake/connector/nanoarrow_cpp/ArrowIterator/FixedSizeListConverter.cpp +73 -0
  76. snowflake/connector/nanoarrow_cpp/ArrowIterator/FixedSizeListConverter.hpp +28 -0
  77. snowflake/connector/nanoarrow_cpp/ArrowIterator/FloatConverter.cpp +30 -0
  78. snowflake/connector/nanoarrow_cpp/ArrowIterator/FloatConverter.hpp +35 -0
  79. snowflake/connector/nanoarrow_cpp/ArrowIterator/IColumnConverter.hpp +17 -0
  80. snowflake/connector/nanoarrow_cpp/ArrowIterator/IntConverter.cpp +23 -0
  81. snowflake/connector/nanoarrow_cpp/ArrowIterator/IntConverter.hpp +45 -0
  82. snowflake/connector/nanoarrow_cpp/ArrowIterator/IntervalConverter.cpp +71 -0
  83. snowflake/connector/nanoarrow_cpp/ArrowIterator/IntervalConverter.hpp +56 -0
  84. snowflake/connector/nanoarrow_cpp/ArrowIterator/LICENSE.txt +209 -0
  85. snowflake/connector/nanoarrow_cpp/ArrowIterator/MapConverter.cpp +75 -0
  86. snowflake/connector/nanoarrow_cpp/ArrowIterator/MapConverter.hpp +30 -0
  87. snowflake/connector/nanoarrow_cpp/ArrowIterator/ObjectConverter.cpp +46 -0
  88. snowflake/connector/nanoarrow_cpp/ArrowIterator/ObjectConverter.hpp +29 -0
  89. snowflake/connector/nanoarrow_cpp/ArrowIterator/Python/Common.cpp +8 -0
  90. snowflake/connector/nanoarrow_cpp/ArrowIterator/Python/Common.hpp +95 -0
  91. snowflake/connector/nanoarrow_cpp/ArrowIterator/Python/Helpers.cpp +57 -0
  92. snowflake/connector/nanoarrow_cpp/ArrowIterator/Python/Helpers.hpp +36 -0
  93. snowflake/connector/nanoarrow_cpp/ArrowIterator/SnowflakeType.cpp +34 -0
  94. snowflake/connector/nanoarrow_cpp/ArrowIterator/SnowflakeType.hpp +48 -0
  95. snowflake/connector/nanoarrow_cpp/ArrowIterator/StringConverter.cpp +19 -0
  96. snowflake/connector/nanoarrow_cpp/ArrowIterator/StringConverter.hpp +26 -0
  97. snowflake/connector/nanoarrow_cpp/ArrowIterator/TimeConverter.cpp +36 -0
  98. snowflake/connector/nanoarrow_cpp/ArrowIterator/TimeConverter.hpp +31 -0
  99. snowflake/connector/nanoarrow_cpp/ArrowIterator/TimeStampConverter.cpp +346 -0
  100. snowflake/connector/nanoarrow_cpp/ArrowIterator/TimeStampConverter.hpp +145 -0
  101. snowflake/connector/nanoarrow_cpp/ArrowIterator/Util/macros.hpp +14 -0
  102. snowflake/connector/nanoarrow_cpp/ArrowIterator/Util/time.cpp +65 -0
  103. snowflake/connector/nanoarrow_cpp/ArrowIterator/Util/time.hpp +68 -0
  104. snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/flatcc_accessors.h +101 -0
  105. snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/flatcc_alloc.h +127 -0
  106. snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/flatcc_assert.h +45 -0
  107. snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/flatcc_builder.h +1908 -0
  108. snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/flatcc_emitter.h +215 -0
  109. snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/flatcc_endian.h +125 -0
  110. snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/flatcc_epilogue.h +7 -0
  111. snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/flatcc_flatbuffers.h +55 -0
  112. snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/flatcc_identifier.h +148 -0
  113. snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/flatcc_iov.h +31 -0
  114. snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/flatcc_prologue.h +8 -0
  115. snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/flatcc_refmap.h +144 -0
  116. snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/flatcc_rtconfig.h +162 -0
  117. snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/flatcc_types.h +97 -0
  118. snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/flatcc_verifier.h +239 -0
  119. snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/portable/flatcc_portable.h +14 -0
  120. snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/portable/paligned_alloc.h +210 -0
  121. snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/portable/pattributes.h +84 -0
  122. snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/portable/pdiagnostic.h +84 -0
  123. snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/portable/pdiagnostic_pop.h +20 -0
  124. snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/portable/pdiagnostic_push.h +51 -0
  125. snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/portable/pendian.h +206 -0
  126. snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/portable/pendian_detect.h +118 -0
  127. snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/portable/pinline.h +19 -0
  128. snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/portable/pinttypes.h +52 -0
  129. snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/portable/portable.h +2 -0
  130. snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/portable/portable_basic.h +25 -0
  131. snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/portable/pstatic_assert.h +67 -0
  132. snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/portable/pstdalign.h +162 -0
  133. snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/portable/pstdint.h +898 -0
  134. snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/portable/punaligned.h +190 -0
  135. snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/portable/pversion.h +6 -0
  136. snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/portable/pwarnings.h +52 -0
  137. snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc.c +3204 -0
  138. snowflake/connector/nanoarrow_cpp/ArrowIterator/nanoarrow.c +3217 -0
  139. snowflake/connector/nanoarrow_cpp/ArrowIterator/nanoarrow.h +3618 -0
  140. snowflake/connector/nanoarrow_cpp/ArrowIterator/nanoarrow.hpp +379 -0
  141. snowflake/connector/nanoarrow_cpp/ArrowIterator/nanoarrow_arrow_iterator.pyx +256 -0
  142. snowflake/connector/nanoarrow_cpp/ArrowIterator/nanoarrow_device.c +512 -0
  143. snowflake/connector/nanoarrow_cpp/ArrowIterator/nanoarrow_device.h +350 -0
  144. snowflake/connector/nanoarrow_cpp/ArrowIterator/nanoarrow_ipc.c +33273 -0
  145. snowflake/connector/nanoarrow_cpp/ArrowIterator/nanoarrow_ipc.h +397 -0
  146. snowflake/connector/nanoarrow_cpp/Logging/logging.cpp +100 -0
  147. snowflake/connector/nanoarrow_cpp/Logging/logging.hpp +49 -0
  148. snowflake/connector/network.py +1297 -0
  149. snowflake/connector/ocsp_asn1crypto.py +447 -0
  150. snowflake/connector/ocsp_snowflake.py +1883 -0
  151. snowflake/connector/options.py +131 -0
  152. snowflake/connector/pandas_tools.py +732 -0
  153. snowflake/connector/proxy.py +43 -0
  154. snowflake/connector/py.typed +0 -0
  155. snowflake/connector/result_batch.py +786 -0
  156. snowflake/connector/result_set.py +319 -0
  157. snowflake/connector/s3_storage_client.py +605 -0
  158. snowflake/connector/secret_detector.py +181 -0
  159. snowflake/connector/sf_dirs.py +64 -0
  160. snowflake/connector/sfbinaryformat.py +35 -0
  161. snowflake/connector/sfdatetime.py +345 -0
  162. snowflake/connector/snow_logging.py +144 -0
  163. snowflake/connector/sqlstate.py +9 -0
  164. snowflake/connector/ssd_internal_keys.py +30 -0
  165. snowflake/connector/ssl_wrap_socket.py +134 -0
  166. snowflake/connector/storage_client.py +478 -0
  167. snowflake/connector/telemetry.py +249 -0
  168. snowflake/connector/telemetry_oob.py +543 -0
  169. snowflake/connector/test_util.py +30 -0
  170. snowflake/connector/time_util.py +159 -0
  171. snowflake/connector/token_cache.py +402 -0
  172. snowflake/connector/tool/__init__.py +0 -0
  173. snowflake/connector/tool/dump_certs.py +57 -0
  174. snowflake/connector/tool/dump_ocsp_response.py +139 -0
  175. snowflake/connector/tool/dump_ocsp_response_cache.py +194 -0
  176. snowflake/connector/tool/probe_connection.py +69 -0
  177. snowflake/connector/url_util.py +49 -0
  178. snowflake/connector/util_text.py +301 -0
  179. snowflake/connector/vendored/__init__.py +3 -0
  180. snowflake/connector/vendored/requests/LICENSE +175 -0
  181. snowflake/connector/vendored/requests/__init__.py +169 -0
  182. snowflake/connector/vendored/requests/__version__.py +14 -0
  183. snowflake/connector/vendored/requests/_internal_utils.py +50 -0
  184. snowflake/connector/vendored/requests/adapters.py +537 -0
  185. snowflake/connector/vendored/requests/api.py +157 -0
  186. snowflake/connector/vendored/requests/auth.py +315 -0
  187. snowflake/connector/vendored/requests/certs.py +17 -0
  188. snowflake/connector/vendored/requests/compat.py +79 -0
  189. snowflake/connector/vendored/requests/cookies.py +561 -0
  190. snowflake/connector/vendored/requests/exceptions.py +140 -0
  191. snowflake/connector/vendored/requests/help.py +134 -0
  192. snowflake/connector/vendored/requests/hooks.py +33 -0
  193. snowflake/connector/vendored/requests/models.py +1033 -0
  194. snowflake/connector/vendored/requests/sessions.py +833 -0
  195. snowflake/connector/vendored/requests/status_codes.py +128 -0
  196. snowflake/connector/vendored/requests/structures.py +99 -0
  197. snowflake/connector/vendored/requests/utils.py +1093 -0
  198. snowflake/connector/vendored/urllib3/LICENSE.txt +21 -0
  199. snowflake/connector/vendored/urllib3/__init__.py +85 -0
  200. snowflake/connector/vendored/urllib3/_collections.py +355 -0
  201. snowflake/connector/vendored/urllib3/_version.py +2 -0
  202. snowflake/connector/vendored/urllib3/connection.py +572 -0
  203. snowflake/connector/vendored/urllib3/connectionpool.py +1137 -0
  204. snowflake/connector/vendored/urllib3/contrib/__init__.py +0 -0
  205. snowflake/connector/vendored/urllib3/contrib/_appengine_environ.py +36 -0
  206. snowflake/connector/vendored/urllib3/contrib/_securetransport/__init__.py +0 -0
  207. snowflake/connector/vendored/urllib3/contrib/_securetransport/bindings.py +519 -0
  208. snowflake/connector/vendored/urllib3/contrib/_securetransport/low_level.py +397 -0
  209. snowflake/connector/vendored/urllib3/contrib/appengine.py +314 -0
  210. snowflake/connector/vendored/urllib3/contrib/ntlmpool.py +130 -0
  211. snowflake/connector/vendored/urllib3/contrib/pyopenssl.py +509 -0
  212. snowflake/connector/vendored/urllib3/contrib/securetransport.py +920 -0
  213. snowflake/connector/vendored/urllib3/contrib/socks.py +216 -0
  214. snowflake/connector/vendored/urllib3/exceptions.py +323 -0
  215. snowflake/connector/vendored/urllib3/fields.py +274 -0
  216. snowflake/connector/vendored/urllib3/filepost.py +98 -0
  217. snowflake/connector/vendored/urllib3/packages/__init__.py +0 -0
  218. snowflake/connector/vendored/urllib3/packages/backports/__init__.py +0 -0
  219. snowflake/connector/vendored/urllib3/packages/backports/makefile.py +51 -0
  220. snowflake/connector/vendored/urllib3/packages/backports/weakref_finalize.py +155 -0
  221. snowflake/connector/vendored/urllib3/packages/six.py +1076 -0
  222. snowflake/connector/vendored/urllib3/poolmanager.py +540 -0
  223. snowflake/connector/vendored/urllib3/request.py +191 -0
  224. snowflake/connector/vendored/urllib3/response.py +885 -0
  225. snowflake/connector/vendored/urllib3/util/__init__.py +49 -0
  226. snowflake/connector/vendored/urllib3/util/connection.py +156 -0
  227. snowflake/connector/vendored/urllib3/util/proxy.py +57 -0
  228. snowflake/connector/vendored/urllib3/util/queue.py +22 -0
  229. snowflake/connector/vendored/urllib3/util/request.py +146 -0
  230. snowflake/connector/vendored/urllib3/util/response.py +107 -0
  231. snowflake/connector/vendored/urllib3/util/retry.py +620 -0
  232. snowflake/connector/vendored/urllib3/util/ssl_.py +495 -0
  233. snowflake/connector/vendored/urllib3/util/ssl_match_hostname.py +159 -0
  234. snowflake/connector/vendored/urllib3/util/ssltransport.py +221 -0
  235. snowflake/connector/vendored/urllib3/util/timeout.py +271 -0
  236. snowflake/connector/vendored/urllib3/util/url.py +435 -0
  237. snowflake/connector/vendored/urllib3/util/wait.py +152 -0
  238. snowflake/connector/version.py +3 -0
  239. snowflake/connector/wif_util.py +407 -0
  240. snowflake_connector_python-3.16.0.dist-info/METADATA +1475 -0
  241. snowflake_connector_python-3.16.0.dist-info/RECORD +246 -0
  242. snowflake_connector_python-3.16.0.dist-info/WHEEL +5 -0
  243. snowflake_connector_python-3.16.0.dist-info/entry_points.txt +4 -0
  244. snowflake_connector_python-3.16.0.dist-info/licenses/LICENSE.txt +202 -0
  245. snowflake_connector_python-3.16.0.dist-info/licenses/NOTICE +8 -0
  246. snowflake_connector_python-3.16.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,127 @@
1
+ #ifndef PC_ARROWTABLEITERATOR_HPP
2
+ #define PC_ARROWTABLEITERATOR_HPP
3
+
4
+ #include <memory>
5
+ #include <string>
6
+ #include <vector>
7
+
8
+ #include "CArrowIterator.hpp"
9
+ #include "nanoarrow.h"
10
+ #include "nanoarrow.hpp"
11
+
12
+ namespace sf {
13
+
14
+ /**
15
+ * Arrow table iterator implementation in C++.
16
+ * The caller will ask for an Arrow Table to be returned back to Python
17
+ * This conversion is zero-copy, just aggregate every columns from multiple
18
+ * record batches and build a new table.
19
+ */
20
+ class CArrowTableIterator : public CArrowIterator {
21
+ public:
22
+ /**
23
+ * Constructor
24
+ */
25
+ CArrowTableIterator(PyObject* context, char* arrow_bytes,
26
+ int64_t arrow_bytes_size, bool number_to_decimal);
27
+
28
+ /**
29
+ * Destructor
30
+ */
31
+ ~CArrowTableIterator() = default;
32
+
33
+ /**
34
+ * @return an arrow table containing all data in all record batches
35
+ */
36
+ ReturnVal next() override;
37
+ std::vector<uintptr_t> getArrowArrayPtrs() override;
38
+ std::vector<uintptr_t> getArrowSchemaPtrs() override;
39
+
40
+ private:
41
+ // nanoarrow data
42
+ std::vector<nanoarrow::UniqueSchema> m_ipcSchemaArrayVec;
43
+
44
+ bool m_tableConverted = false;
45
+
46
+ /** arrow format convert context for the current session */
47
+ PyObject* m_context;
48
+
49
+ /** local time zone */
50
+ char* m_timezone;
51
+ const bool m_convert_number_to_decimal;
52
+
53
+ /**
54
+ * Reconstruct record batches with type conversion in place
55
+ */
56
+ void reconstructRecordBatches();
57
+ void convertIfNeeded(ArrowSchema* columnSchema, ArrowArrayView* columnArray);
58
+ void reconstructRecordBatches_nanoarrow();
59
+
60
+ /**
61
+ * Convert all current RecordBatches to Arrow Table
62
+ * @return if conversion is executed at first time and successfully
63
+ */
64
+ bool convertRecordBatchesToTable_nanoarrow();
65
+
66
+ /**
67
+ * convert scaled fixed number column to Decimal, or Double column based on
68
+ * setting
69
+ */
70
+ void convertScaledFixedNumberColumn_nanoarrow(ArrowSchemaView* field,
71
+ ArrowArrayView* columnArray,
72
+ const unsigned int scale);
73
+
74
+ /**
75
+ * convert scaled fixed number column to Decimal column
76
+ */
77
+ void convertScaledFixedNumberColumnToDecimalColumn_nanoarrow(
78
+ ArrowSchemaView* field, ArrowArrayView* columnArray,
79
+ const unsigned int scale);
80
+
81
+ /**
82
+ * convert scaled fixed number column to Double column
83
+ */
84
+ void convertScaledFixedNumberColumnToDoubleColumn_nanoarrow(
85
+ ArrowSchemaView* field, ArrowArrayView* columnArray,
86
+ const unsigned int scale);
87
+
88
+ /**
89
+ * convert Snowflake Time column (Arrow int32/int64) to Arrow Time column
90
+ * Since Python/Pandas Time does not support nanoseconds, this function
91
+ * truncates values to microseconds if necessary
92
+ */
93
+ void convertTimeColumn_nanoarrow(ArrowSchemaView* field,
94
+ ArrowArrayView* columnArray,
95
+ const int scale);
96
+
97
+ /**
98
+ * convert Snowflake TimestampNTZ/TimestampLTZ column to Arrow Timestamp
99
+ * column
100
+ */
101
+ void convertTimestampColumn_nanoarrow(ArrowSchemaView* field,
102
+ ArrowArrayView* columnArray,
103
+ const int scale,
104
+ const std::string timezone = "");
105
+
106
+ /**
107
+ * convert Snowflake TimestampTZ column to Arrow Timestamp column in UTC
108
+ * Arrow Timestamp does not support time zone info in each value, so this
109
+ * method convert TimestampTZ to Arrow timestamp with UTC timezone
110
+ */
111
+ void convertTimestampTZColumn_nanoarrow(ArrowSchemaView* field,
112
+ ArrowArrayView* columnArray,
113
+ const int scale, const int byteLength,
114
+ const std::string timezone);
115
+
116
+ /**
117
+ * convert scaled fixed number to double
118
+ * if scale is small, then just divide based on the scale; otherwise, convert
119
+ * the value to string first and then convert to double to avoid precision
120
+ * loss
121
+ */
122
+ template <typename T>
123
+ double convertScaledFixedNumberToDouble(const unsigned int scale,
124
+ T originalValue);
125
+ };
126
+ } // namespace sf
127
+ #endif // PC_ARROWTABLEITERATOR_HPP
@@ -0,0 +1,47 @@
1
+ #include "DateConverter.hpp"
2
+
3
+ #include <memory>
4
+
5
+ #include "Python/Helpers.hpp"
6
+
7
+ namespace sf {
8
+ Logger* DateConverter::logger = new Logger("snowflake.connector.DateConverter");
9
+
10
+ py::UniqueRef& DateConverter::initPyDatetimeDate() {
11
+ static py::UniqueRef pyDatetimeDate;
12
+ if (pyDatetimeDate.empty()) {
13
+ py::UniqueRef pyDatetimeModule;
14
+ py::importPythonModule("datetime", pyDatetimeModule);
15
+ py::importFromModule(pyDatetimeModule, "date", pyDatetimeDate);
16
+ Py_XINCREF(pyDatetimeDate.get());
17
+ }
18
+ return pyDatetimeDate;
19
+ }
20
+
21
+ DateConverter::DateConverter(ArrowArrayView* array)
22
+ : m_array(array), m_pyDatetimeDate(initPyDatetimeDate()) {}
23
+
24
+ PyObject* DateConverter::toPyObject(int64_t rowIndex) const {
25
+ if (ArrowArrayViewIsNull(m_array, rowIndex)) {
26
+ Py_RETURN_NONE;
27
+ }
28
+
29
+ int64_t deltaDays = ArrowArrayViewGetIntUnsafe(m_array, rowIndex);
30
+ return PyObject_CallMethod(m_pyDatetimeDate.get(), "fromordinal", "i",
31
+ epochDay + deltaDays);
32
+ }
33
+
34
+ NumpyDateConverter::NumpyDateConverter(ArrowArrayView* array, PyObject* context)
35
+ : m_array(array), m_context(context) {}
36
+
37
+ PyObject* NumpyDateConverter::toPyObject(int64_t rowIndex) const {
38
+ if (ArrowArrayViewIsNull(m_array, rowIndex)) {
39
+ Py_RETURN_NONE;
40
+ }
41
+
42
+ int64_t deltaDays = ArrowArrayViewGetIntUnsafe(m_array, rowIndex);
43
+ return PyObject_CallMethod(m_context, "DATE_to_numpy_datetime64", "i",
44
+ deltaDays);
45
+ }
46
+
47
+ } // namespace sf
@@ -0,0 +1,46 @@
1
+ #ifndef PC_DATECONVERTER_HPP
2
+ #define PC_DATECONVERTER_HPP
3
+
4
+ #include <memory>
5
+
6
+ #include "IColumnConverter.hpp"
7
+ #include "Python/Common.hpp"
8
+ #include "logging.hpp"
9
+ #include "nanoarrow.h"
10
+
11
+ namespace sf {
12
+
13
+ class DateConverter : public IColumnConverter {
14
+ public:
15
+ explicit DateConverter(ArrowArrayView* array);
16
+
17
+ PyObject* toPyObject(int64_t rowIndex) const override;
18
+
19
+ private:
20
+ static py::UniqueRef& initPyDatetimeDate();
21
+
22
+ ArrowArrayView* m_array;
23
+
24
+ /** from Python Ordinal to 1970-01-01 */
25
+ static constexpr int epochDay = 719163;
26
+
27
+ static Logger* logger;
28
+
29
+ py::UniqueRef& m_pyDatetimeDate;
30
+ };
31
+
32
+ class NumpyDateConverter : public IColumnConverter {
33
+ public:
34
+ explicit NumpyDateConverter(ArrowArrayView* array, PyObject* context);
35
+
36
+ PyObject* toPyObject(int64_t rowIndex) const override;
37
+
38
+ private:
39
+ ArrowArrayView* m_array;
40
+
41
+ PyObject* m_context;
42
+ };
43
+
44
+ } // namespace sf
45
+
46
+ #endif // PC_DATECONVERTER_HPP
@@ -0,0 +1,83 @@
1
+
2
+ #include "DecFloatConverter.hpp"
3
+
4
+ #include <cstring>
5
+ #include <memory>
6
+
7
+ #include "Python/Helpers.hpp"
8
+
9
+ namespace sf {
10
+
11
+ Logger* DecFloatConverter::logger =
12
+ new Logger("snowflake.connector.DecFloatConverter");
13
+
14
+ const std::string DecFloatConverter::FIELD_NAME_EXPONENT = "exponent";
15
+ const std::string DecFloatConverter::FIELD_NAME_SIGNIFICAND = "significand";
16
+
17
+ DecFloatConverter::DecFloatConverter(ArrowArrayView& array,
18
+ ArrowSchemaView& schema, PyObject& context,
19
+ bool useNumpy)
20
+ : m_context(context),
21
+ m_array(array),
22
+ m_exponent(nullptr),
23
+ m_significand(nullptr),
24
+ m_useNumpy(useNumpy) {
25
+ if (schema.schema->n_children != 2) {
26
+ std::string errorInfo = Logger::formatString(
27
+ "[Snowflake Exception] arrow schema field number does not match, "
28
+ "expected 2 but got %d instead",
29
+ schema.schema->n_children);
30
+ logger->error(__FILE__, __func__, __LINE__, errorInfo.c_str());
31
+ PyErr_SetString(PyExc_Exception, errorInfo.c_str());
32
+ return;
33
+ }
34
+ for (int i = 0; i < schema.schema->n_children; i += 1) {
35
+ ArrowSchema* c_schema = schema.schema->children[i];
36
+ if (std::strcmp(c_schema->name,
37
+ DecFloatConverter::FIELD_NAME_EXPONENT.c_str()) == 0) {
38
+ m_exponent = m_array.children[i];
39
+ } else if (std::strcmp(c_schema->name,
40
+ DecFloatConverter::FIELD_NAME_SIGNIFICAND.c_str()) ==
41
+ 0) {
42
+ m_significand = m_array.children[i];
43
+ }
44
+ }
45
+ if (!m_exponent || !m_significand) {
46
+ std::string errorInfo = Logger::formatString(
47
+ "[Snowflake Exception] arrow schema field names do not match, "
48
+ "expected %s and %s, but got %s and %s instead",
49
+ DecFloatConverter::FIELD_NAME_EXPONENT.c_str(),
50
+ DecFloatConverter::FIELD_NAME_SIGNIFICAND.c_str(),
51
+ schema.schema->children[0]->name, schema.schema->children[1]->name);
52
+ logger->error(__FILE__, __func__, __LINE__, errorInfo.c_str());
53
+ PyErr_SetString(PyExc_Exception, errorInfo.c_str());
54
+ return;
55
+ }
56
+ }
57
+
58
+ PyObject* DecFloatConverter::toPyObject(int64_t rowIndex) const {
59
+ if (ArrowArrayViewIsNull(&m_array, rowIndex)) {
60
+ Py_RETURN_NONE;
61
+ }
62
+ int64_t exponent = ArrowArrayViewGetIntUnsafe(m_exponent, rowIndex);
63
+ ArrowStringView stringView =
64
+ ArrowArrayViewGetStringUnsafe(m_significand, rowIndex);
65
+ if (stringView.size_bytes > 16) {
66
+ std::string errorInfo = Logger::formatString(
67
+ "[Snowflake Exception] only precisions up to 38 supported. "
68
+ "Please update to a newer version of the connector.");
69
+ logger->error(__FILE__, __func__, __LINE__, errorInfo.c_str());
70
+ PyErr_SetString(PyExc_Exception, errorInfo.c_str());
71
+ return nullptr;
72
+ }
73
+ PyObject* significand =
74
+ PyBytes_FromStringAndSize(stringView.data, stringView.size_bytes);
75
+
76
+ PyObject* result = PyObject_CallMethod(
77
+ &m_context,
78
+ m_useNumpy ? "DECFLOAT_to_numpy_float64" : "DECFLOAT_to_decimal", "iS",
79
+ exponent, significand);
80
+ Py_XDECREF(significand);
81
+ return result;
82
+ }
83
+ } // namespace sf
@@ -0,0 +1,35 @@
1
+
2
+ #ifndef PC_DECFLOATCONVERTER_HPP
3
+ #define PC_DECFLOATCONVERTER_HPP
4
+
5
+ #include <memory>
6
+
7
+ #include "IColumnConverter.hpp"
8
+ #include "logging.hpp"
9
+ #include "nanoarrow.h"
10
+
11
+ namespace sf {
12
+
13
+ class DecFloatConverter : public IColumnConverter {
14
+ public:
15
+ const static std::string FIELD_NAME_EXPONENT;
16
+ const static std::string FIELD_NAME_SIGNIFICAND;
17
+
18
+ explicit DecFloatConverter(ArrowArrayView& array, ArrowSchemaView& schema,
19
+ PyObject& context, bool useNumpy);
20
+
21
+ PyObject* toPyObject(int64_t rowIndex) const override;
22
+
23
+ private:
24
+ PyObject& m_context;
25
+ ArrowArrayView& m_array;
26
+ ArrowArrayView* m_exponent;
27
+ ArrowArrayView* m_significand;
28
+ bool m_useNumpy;
29
+
30
+ static Logger* logger;
31
+ };
32
+
33
+ } // namespace sf
34
+
35
+ #endif // PC_DECFLOATCONVERTER_HPP
@@ -0,0 +1,97 @@
1
+ #include "DecimalConverter.hpp"
2
+
3
+ #include <memory>
4
+ #include <string>
5
+
6
+ #include "Python/Common.hpp"
7
+ #include "Python/Helpers.hpp"
8
+
9
+ namespace sf {
10
+
11
+ DecimalBaseConverter::DecimalBaseConverter()
12
+ : m_pyDecimalConstructor(initPyDecimalConstructor()) {}
13
+
14
+ py::UniqueRef& DecimalBaseConverter::initPyDecimalConstructor() {
15
+ static py::UniqueRef pyDecimalConstructor;
16
+ if (pyDecimalConstructor.empty()) {
17
+ py::UniqueRef decimalModule;
18
+ py::importPythonModule("decimal", decimalModule);
19
+ py::importFromModule(decimalModule, "Decimal", pyDecimalConstructor);
20
+ Py_XINCREF(pyDecimalConstructor.get());
21
+ }
22
+
23
+ return pyDecimalConstructor;
24
+ }
25
+
26
+ DecimalFromIntConverter::DecimalFromIntConverter(ArrowArrayView* array,
27
+ int precision, int scale)
28
+ : m_array(array), m_precision(precision), m_scale(scale) {}
29
+
30
+ PyObject* DecimalFromIntConverter::toPyObject(int64_t rowIndex) const {
31
+ if (ArrowArrayViewIsNull(m_array, rowIndex)) {
32
+ Py_RETURN_NONE;
33
+ }
34
+ int64_t val = ArrowArrayViewGetIntUnsafe(m_array, rowIndex);
35
+ py::UniqueRef decimal(
36
+ PyObject_CallFunction(m_pyDecimalConstructor.get(), "L", val));
37
+ return PyObject_CallMethod(decimal.get(), "scaleb", "i", -m_scale);
38
+ }
39
+
40
+ NumpyDecimalConverter::NumpyDecimalConverter(ArrowArrayView* array,
41
+ int precision, int scale,
42
+ PyObject* context)
43
+ : m_array(array),
44
+ m_precision(precision),
45
+ m_scale(scale),
46
+ m_context(context) {}
47
+
48
+ PyObject* NumpyDecimalConverter::toPyObject(int64_t rowIndex) const {
49
+ if (ArrowArrayViewIsNull(m_array, rowIndex)) {
50
+ Py_RETURN_NONE;
51
+ }
52
+ int64_t val = ArrowArrayViewGetIntUnsafe(m_array, rowIndex);
53
+ return PyObject_CallMethod(m_context, "FIXED_to_numpy_float64", "Li", val,
54
+ m_scale);
55
+ }
56
+
57
+ DecimalFromDecimalConverter::DecimalFromDecimalConverter(PyObject* context,
58
+ ArrowArrayView* array,
59
+ int scale)
60
+ : m_array(array), m_context(context), m_scale(scale) {}
61
+
62
+ PyObject* DecimalFromDecimalConverter::toPyObject(int64_t rowIndex) const {
63
+ if (ArrowArrayViewIsNull(m_array, rowIndex)) {
64
+ Py_RETURN_NONE;
65
+ }
66
+ int64_t bytes_start = 16 * (m_array->array->offset + rowIndex);
67
+ const char* ptr_start = m_array->buffer_views[1].data.as_char;
68
+ PyObject* int128_bytes =
69
+ PyBytes_FromStringAndSize(&(ptr_start[bytes_start]), 16);
70
+ /**
71
+ # Alternatively, the decimal conversion can be implemented using the
72
+ ArrowDecimal related APIs in the following # code snippets, however, it's less
73
+ performant than the direct memory manipulation. # The code snippets here is
74
+ for context reference.
75
+
76
+ ArrowDecimal arrowDecimal;
77
+ ArrowDecimalInit(&arrowDecimal, 128, precision, scale);
78
+ ArrowArrayViewGetDecimalUnsafe(m_array, rowIndex, &arrowDecimal);
79
+ uint8_t outBytes[16];
80
+ ArrowDecimalGetBytes(&arrowDecimal, outBytes);
81
+ PyObject* int128_bytes = PyBytes_FromStringAndSize(&outBytes, 16);
82
+ */
83
+ PyObject* return_object = PyObject_CallMethod(
84
+ m_context, "DECIMAL128_to_decimal", "Si", int128_bytes, m_scale);
85
+ /**
86
+ int128_bytes is a new referenced created by PyBytes_FromStringAndSize,
87
+ to avoid memory leak we need to free it after usage
88
+ check docs:
89
+ https://docs.python.org/3/c-api/bytes.html#c.PyBytes_FromStringAndSize
90
+ https://docs.python.org/3/c-api/refcounting.html#c.Py_XDECREF
91
+
92
+ */
93
+ Py_XDECREF(int128_bytes);
94
+ return return_object;
95
+ }
96
+
97
+ } // namespace sf
@@ -0,0 +1,72 @@
1
+ #ifndef PC_DECIMALCONVERTER_HPP
2
+ #define PC_DECIMALCONVERTER_HPP
3
+
4
+ #include <memory>
5
+
6
+ #include "IColumnConverter.hpp"
7
+ #include "Python/Common.hpp"
8
+ #include "nanoarrow.h"
9
+
10
+ namespace sf {
11
+
12
+ class DecimalBaseConverter : public IColumnConverter {
13
+ public:
14
+ DecimalBaseConverter();
15
+ virtual ~DecimalBaseConverter() = default;
16
+
17
+ protected:
18
+ py::UniqueRef& m_pyDecimalConstructor;
19
+
20
+ private:
21
+ static py::UniqueRef& initPyDecimalConstructor();
22
+ };
23
+
24
+ class DecimalFromDecimalConverter : public DecimalBaseConverter {
25
+ public:
26
+ explicit DecimalFromDecimalConverter(PyObject* context, ArrowArrayView* array,
27
+ int scale);
28
+
29
+ PyObject* toPyObject(int64_t rowIndex) const override;
30
+
31
+ private:
32
+ ArrowArrayView* m_array;
33
+ PyObject* m_context;
34
+ int m_scale;
35
+ /** no need for this converter to store precision*/
36
+ };
37
+
38
+ class DecimalFromIntConverter : public DecimalBaseConverter {
39
+ public:
40
+ explicit DecimalFromIntConverter(ArrowArrayView* array, int precision,
41
+ int scale);
42
+
43
+ PyObject* toPyObject(int64_t rowIndex) const override;
44
+
45
+ private:
46
+ ArrowArrayView* m_array;
47
+ int m_precision; // looks like the precision here is not useful, and this
48
+ // will be removed soon when it's been confirmed
49
+
50
+ int m_scale;
51
+ };
52
+
53
+ class NumpyDecimalConverter : public IColumnConverter {
54
+ public:
55
+ explicit NumpyDecimalConverter(ArrowArrayView* array, int precision,
56
+ int scale, PyObject* context);
57
+
58
+ PyObject* toPyObject(int64_t rowIndex) const override;
59
+
60
+ private:
61
+ ArrowArrayView* m_array;
62
+
63
+ int m_precision;
64
+
65
+ int m_scale;
66
+
67
+ PyObject* m_context;
68
+ };
69
+
70
+ } // namespace sf
71
+
72
+ #endif // PC_DECIMALCONVERTER_HPP
@@ -0,0 +1,73 @@
1
+ #include "FixedSizeListConverter.hpp"
2
+
3
+ namespace sf {
4
+ Logger* FixedSizeListConverter::logger =
5
+ new Logger("snowflake.connector.FixedSizeListConverter");
6
+
7
+ FixedSizeListConverter::FixedSizeListConverter(ArrowArrayView* array)
8
+ : m_array(array) {}
9
+
10
+ void FixedSizeListConverter::generateError(const std::string& msg) const {
11
+ logger->error(__FILE__, __func__, __LINE__, msg.c_str());
12
+ PyErr_SetString(PyExc_Exception, msg.c_str());
13
+ }
14
+
15
+ PyObject* FixedSizeListConverter::toPyObject(int64_t rowIndex) const {
16
+ if (ArrowArrayViewIsNull(m_array, rowIndex)) {
17
+ Py_RETURN_NONE;
18
+ }
19
+
20
+ if (m_array->n_children != 1) {
21
+ std::string errorInfo = Logger::formatString(
22
+ "[Snowflake Exception] invalid arrow element schema for fixed size "
23
+ "list: got (%d) "
24
+ "children",
25
+ m_array->n_children);
26
+ this->generateError(errorInfo);
27
+ return nullptr;
28
+ }
29
+
30
+ // m_array->length represents the number of fixed size lists in the array
31
+ // m_array->children[0] has a buffer view that contains the actual data of
32
+ // each list, back-to-back m_array->children[0]->length represents the sum of
33
+ // the lengths of the fixed size lists in the array.
34
+
35
+ ArrowArrayView* elements = m_array->children[0];
36
+ const auto fixedSizeArrayLength = elements->length / m_array->length;
37
+ PyObject* list = PyList_New(fixedSizeArrayLength);
38
+
39
+ const int64_t startIndexWithoutOffset = rowIndex * fixedSizeArrayLength;
40
+ for (int64_t i = 0; i < fixedSizeArrayLength; ++i) {
41
+ const auto bufferIndexWithoutOffset = startIndexWithoutOffset + i;
42
+ // Currently, the backend only sends back INT32 and FLOAT32, but the
43
+ // remaining types are enumerated for future use.
44
+ switch (elements->storage_type) {
45
+ case NANOARROW_TYPE_INT8:
46
+ case NANOARROW_TYPE_INT16:
47
+ case NANOARROW_TYPE_INT32:
48
+ case NANOARROW_TYPE_INT64: {
49
+ const auto value =
50
+ ArrowArrayViewGetIntUnsafe(elements, bufferIndexWithoutOffset);
51
+ PyList_SetItem(list, i, PyLong_FromLongLong(value));
52
+ } break;
53
+ case NANOARROW_TYPE_HALF_FLOAT:
54
+ case NANOARROW_TYPE_FLOAT:
55
+ case NANOARROW_TYPE_DOUBLE: {
56
+ const auto value =
57
+ ArrowArrayViewGetDoubleUnsafe(elements, bufferIndexWithoutOffset);
58
+ PyList_SetItem(list, i, PyFloat_FromDouble(value));
59
+ } break;
60
+ default:
61
+ std::string errorInfo = Logger::formatString(
62
+ "[Snowflake Exception] invalid arrow element type for fixed size "
63
+ "list: got (%s)",
64
+ ArrowTypeString(elements->storage_type));
65
+ this->generateError(errorInfo);
66
+ return nullptr;
67
+ }
68
+ }
69
+
70
+ return list;
71
+ }
72
+
73
+ } // namespace sf
@@ -0,0 +1,28 @@
1
+ #ifndef PC_FIXEDSIZELISTCONVERTER_HPP
2
+ #define PC_FIXEDSIZELISTCONVERTER_HPP
3
+
4
+ #include <memory>
5
+
6
+ #include "IColumnConverter.hpp"
7
+ #include "logging.hpp"
8
+ #include "nanoarrow.h"
9
+ #include "nanoarrow.hpp"
10
+
11
+ namespace sf {
12
+
13
+ class FixedSizeListConverter : public IColumnConverter {
14
+ public:
15
+ explicit FixedSizeListConverter(ArrowArrayView* array);
16
+ PyObject* toPyObject(int64_t rowIndex) const override;
17
+
18
+ private:
19
+ void generateError(const std::string& msg) const;
20
+
21
+ ArrowArrayView* m_array;
22
+
23
+ static Logger* logger;
24
+ };
25
+
26
+ } // namespace sf
27
+
28
+ #endif // PC_FIXEDSIZELISTCONVERTER_HPP
@@ -0,0 +1,30 @@
1
+ #include "FloatConverter.hpp"
2
+
3
+ #include <memory>
4
+
5
+ namespace sf {
6
+
7
+ /** snowflake float is 64-precision, which refers to double here */
8
+ FloatConverter::FloatConverter(ArrowArrayView* array) : m_array(array) {}
9
+
10
+ PyObject* FloatConverter::toPyObject(int64_t rowIndex) const {
11
+ if (ArrowArrayViewIsNull(m_array, rowIndex)) {
12
+ Py_RETURN_NONE;
13
+ }
14
+ return PyFloat_FromDouble(ArrowArrayViewGetDoubleUnsafe(m_array, rowIndex));
15
+ }
16
+
17
+ NumpyFloat64Converter::NumpyFloat64Converter(ArrowArrayView* array,
18
+ PyObject* context)
19
+ : m_array(array), m_context(context) {}
20
+
21
+ PyObject* NumpyFloat64Converter::toPyObject(int64_t rowIndex) const {
22
+ if (ArrowArrayViewIsNull(m_array, rowIndex)) {
23
+ Py_RETURN_NONE;
24
+ }
25
+
26
+ double val = ArrowArrayViewGetDoubleUnsafe(m_array, rowIndex);
27
+ return PyObject_CallMethod(m_context, "REAL_to_numpy_float64", "d", val);
28
+ }
29
+
30
+ } // namespace sf
@@ -0,0 +1,35 @@
1
+ #ifndef PC_FLOATCONVERTER_HPP
2
+ #define PC_FLOATCONVERTER_HPP
3
+
4
+ #include <memory>
5
+
6
+ #include "IColumnConverter.hpp"
7
+ #include "nanoarrow.h"
8
+
9
+ namespace sf {
10
+
11
+ class FloatConverter : public IColumnConverter {
12
+ public:
13
+ explicit FloatConverter(ArrowArrayView* array);
14
+
15
+ PyObject* toPyObject(int64_t rowIndex) const override;
16
+
17
+ private:
18
+ ArrowArrayView* m_array;
19
+ };
20
+
21
+ class NumpyFloat64Converter : public IColumnConverter {
22
+ public:
23
+ explicit NumpyFloat64Converter(ArrowArrayView* array, PyObject* context);
24
+
25
+ PyObject* toPyObject(int64_t rowIndex) const override;
26
+
27
+ private:
28
+ ArrowArrayView* m_array;
29
+
30
+ PyObject* m_context;
31
+ };
32
+
33
+ } // namespace sf
34
+
35
+ #endif // PC_FLOATCONVERTER_HPP