snowflake-connector-python 3.16.0__cp313-cp313-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- snowflake/connector/__init__.py +96 -0
- snowflake/connector/_query_context_cache.py +276 -0
- snowflake/connector/_sql_util.py +48 -0
- snowflake/connector/_utils.py +70 -0
- snowflake/connector/arrow_context.py +203 -0
- snowflake/connector/auth/__init__.py +53 -0
- snowflake/connector/auth/_auth.py +586 -0
- snowflake/connector/auth/_http_server.py +220 -0
- snowflake/connector/auth/_oauth_base.py +400 -0
- snowflake/connector/auth/by_plugin.py +219 -0
- snowflake/connector/auth/default.py +36 -0
- snowflake/connector/auth/idtoken.py +72 -0
- snowflake/connector/auth/keypair.py +222 -0
- snowflake/connector/auth/no_auth.py +39 -0
- snowflake/connector/auth/oauth.py +49 -0
- snowflake/connector/auth/oauth_code.py +479 -0
- snowflake/connector/auth/oauth_credentials.py +66 -0
- snowflake/connector/auth/okta.py +335 -0
- snowflake/connector/auth/pat.py +39 -0
- snowflake/connector/auth/usrpwdmfa.py +69 -0
- snowflake/connector/auth/webbrowser.py +502 -0
- snowflake/connector/auth/workload_identity.py +94 -0
- snowflake/connector/azure_storage_client.py +276 -0
- snowflake/connector/backoff_policies.py +141 -0
- snowflake/connector/bind_upload_agent.py +89 -0
- snowflake/connector/cache.py +696 -0
- snowflake/connector/compat.py +127 -0
- snowflake/connector/config_manager.py +496 -0
- snowflake/connector/connection.py +2273 -0
- snowflake/connector/connection_diagnostic.py +776 -0
- snowflake/connector/constants.py +442 -0
- snowflake/connector/converter.py +785 -0
- snowflake/connector/converter_issue23517.py +87 -0
- snowflake/connector/converter_null.py +14 -0
- snowflake/connector/converter_snowsql.py +205 -0
- snowflake/connector/cursor.py +1951 -0
- snowflake/connector/dbapi.py +53 -0
- snowflake/connector/description.py +19 -0
- snowflake/connector/direct_file_operation_utils.py +88 -0
- snowflake/connector/encryption_util.py +220 -0
- snowflake/connector/errorcode.py +91 -0
- snowflake/connector/errors.py +615 -0
- snowflake/connector/externals_utils/__init__.py +0 -0
- snowflake/connector/externals_utils/externals_setup.py +27 -0
- snowflake/connector/feature.py +4 -0
- snowflake/connector/file_compression_type.py +118 -0
- snowflake/connector/file_lock.py +72 -0
- snowflake/connector/file_transfer_agent.py +1215 -0
- snowflake/connector/file_util.py +153 -0
- snowflake/connector/gcs_storage_client.py +474 -0
- snowflake/connector/gzip_decoder.py +85 -0
- snowflake/connector/local_storage_client.py +90 -0
- snowflake/connector/log_configuration.py +60 -0
- snowflake/connector/logging_utils/__init__.py +0 -0
- snowflake/connector/logging_utils/filters.py +72 -0
- snowflake/connector/nanoarrow_arrow_iterator.cp313-win_amd64.pyd +0 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/ArrayConverter.cpp +60 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/ArrayConverter.hpp +29 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/BinaryConverter.cpp +19 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/BinaryConverter.hpp +26 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/BooleanConverter.cpp +21 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/BooleanConverter.hpp +23 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/CArrowChunkIterator.cpp +557 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/CArrowChunkIterator.hpp +98 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/CArrowIterator.cpp +125 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/CArrowIterator.hpp +115 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/CArrowTableIterator.cpp +1005 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/CArrowTableIterator.hpp +127 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/DateConverter.cpp +47 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/DateConverter.hpp +46 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/DecFloatConverter.cpp +83 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/DecFloatConverter.hpp +35 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/DecimalConverter.cpp +97 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/DecimalConverter.hpp +72 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/FixedSizeListConverter.cpp +73 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/FixedSizeListConverter.hpp +28 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/FloatConverter.cpp +30 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/FloatConverter.hpp +35 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/IColumnConverter.hpp +17 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/IntConverter.cpp +23 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/IntConverter.hpp +45 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/IntervalConverter.cpp +71 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/IntervalConverter.hpp +56 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/LICENSE.txt +209 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/MapConverter.cpp +75 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/MapConverter.hpp +30 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/ObjectConverter.cpp +46 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/ObjectConverter.hpp +29 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/Python/Common.cpp +8 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/Python/Common.hpp +95 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/Python/Helpers.cpp +57 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/Python/Helpers.hpp +36 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/SnowflakeType.cpp +34 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/SnowflakeType.hpp +48 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/StringConverter.cpp +19 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/StringConverter.hpp +26 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/TimeConverter.cpp +36 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/TimeConverter.hpp +31 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/TimeStampConverter.cpp +346 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/TimeStampConverter.hpp +145 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/Util/macros.hpp +14 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/Util/time.cpp +65 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/Util/time.hpp +68 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/flatcc_accessors.h +101 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/flatcc_alloc.h +127 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/flatcc_assert.h +45 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/flatcc_builder.h +1908 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/flatcc_emitter.h +215 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/flatcc_endian.h +125 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/flatcc_epilogue.h +7 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/flatcc_flatbuffers.h +55 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/flatcc_identifier.h +148 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/flatcc_iov.h +31 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/flatcc_prologue.h +8 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/flatcc_refmap.h +144 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/flatcc_rtconfig.h +162 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/flatcc_types.h +97 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/flatcc_verifier.h +239 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/portable/flatcc_portable.h +14 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/portable/paligned_alloc.h +210 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/portable/pattributes.h +84 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/portable/pdiagnostic.h +84 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/portable/pdiagnostic_pop.h +20 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/portable/pdiagnostic_push.h +51 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/portable/pendian.h +206 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/portable/pendian_detect.h +118 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/portable/pinline.h +19 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/portable/pinttypes.h +52 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/portable/portable.h +2 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/portable/portable_basic.h +25 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/portable/pstatic_assert.h +67 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/portable/pstdalign.h +162 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/portable/pstdint.h +898 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/portable/punaligned.h +190 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/portable/pversion.h +6 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc/portable/pwarnings.h +52 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/flatcc.c +3204 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/nanoarrow.c +3217 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/nanoarrow.h +3618 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/nanoarrow.hpp +379 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/nanoarrow_arrow_iterator.pyx +256 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/nanoarrow_device.c +512 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/nanoarrow_device.h +350 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/nanoarrow_ipc.c +33273 -0
- snowflake/connector/nanoarrow_cpp/ArrowIterator/nanoarrow_ipc.h +397 -0
- snowflake/connector/nanoarrow_cpp/Logging/logging.cpp +100 -0
- snowflake/connector/nanoarrow_cpp/Logging/logging.hpp +49 -0
- snowflake/connector/network.py +1297 -0
- snowflake/connector/ocsp_asn1crypto.py +447 -0
- snowflake/connector/ocsp_snowflake.py +1883 -0
- snowflake/connector/options.py +131 -0
- snowflake/connector/pandas_tools.py +732 -0
- snowflake/connector/proxy.py +43 -0
- snowflake/connector/py.typed +0 -0
- snowflake/connector/result_batch.py +786 -0
- snowflake/connector/result_set.py +319 -0
- snowflake/connector/s3_storage_client.py +605 -0
- snowflake/connector/secret_detector.py +181 -0
- snowflake/connector/sf_dirs.py +64 -0
- snowflake/connector/sfbinaryformat.py +35 -0
- snowflake/connector/sfdatetime.py +345 -0
- snowflake/connector/snow_logging.py +144 -0
- snowflake/connector/sqlstate.py +9 -0
- snowflake/connector/ssd_internal_keys.py +30 -0
- snowflake/connector/ssl_wrap_socket.py +134 -0
- snowflake/connector/storage_client.py +478 -0
- snowflake/connector/telemetry.py +249 -0
- snowflake/connector/telemetry_oob.py +543 -0
- snowflake/connector/test_util.py +30 -0
- snowflake/connector/time_util.py +159 -0
- snowflake/connector/token_cache.py +402 -0
- snowflake/connector/tool/__init__.py +0 -0
- snowflake/connector/tool/dump_certs.py +57 -0
- snowflake/connector/tool/dump_ocsp_response.py +139 -0
- snowflake/connector/tool/dump_ocsp_response_cache.py +194 -0
- snowflake/connector/tool/probe_connection.py +69 -0
- snowflake/connector/url_util.py +49 -0
- snowflake/connector/util_text.py +301 -0
- snowflake/connector/vendored/__init__.py +3 -0
- snowflake/connector/vendored/requests/LICENSE +175 -0
- snowflake/connector/vendored/requests/__init__.py +169 -0
- snowflake/connector/vendored/requests/__version__.py +14 -0
- snowflake/connector/vendored/requests/_internal_utils.py +50 -0
- snowflake/connector/vendored/requests/adapters.py +537 -0
- snowflake/connector/vendored/requests/api.py +157 -0
- snowflake/connector/vendored/requests/auth.py +315 -0
- snowflake/connector/vendored/requests/certs.py +17 -0
- snowflake/connector/vendored/requests/compat.py +79 -0
- snowflake/connector/vendored/requests/cookies.py +561 -0
- snowflake/connector/vendored/requests/exceptions.py +140 -0
- snowflake/connector/vendored/requests/help.py +134 -0
- snowflake/connector/vendored/requests/hooks.py +33 -0
- snowflake/connector/vendored/requests/models.py +1033 -0
- snowflake/connector/vendored/requests/sessions.py +833 -0
- snowflake/connector/vendored/requests/status_codes.py +128 -0
- snowflake/connector/vendored/requests/structures.py +99 -0
- snowflake/connector/vendored/requests/utils.py +1093 -0
- snowflake/connector/vendored/urllib3/LICENSE.txt +21 -0
- snowflake/connector/vendored/urllib3/__init__.py +85 -0
- snowflake/connector/vendored/urllib3/_collections.py +355 -0
- snowflake/connector/vendored/urllib3/_version.py +2 -0
- snowflake/connector/vendored/urllib3/connection.py +572 -0
- snowflake/connector/vendored/urllib3/connectionpool.py +1137 -0
- snowflake/connector/vendored/urllib3/contrib/__init__.py +0 -0
- snowflake/connector/vendored/urllib3/contrib/_appengine_environ.py +36 -0
- snowflake/connector/vendored/urllib3/contrib/_securetransport/__init__.py +0 -0
- snowflake/connector/vendored/urllib3/contrib/_securetransport/bindings.py +519 -0
- snowflake/connector/vendored/urllib3/contrib/_securetransport/low_level.py +397 -0
- snowflake/connector/vendored/urllib3/contrib/appengine.py +314 -0
- snowflake/connector/vendored/urllib3/contrib/ntlmpool.py +130 -0
- snowflake/connector/vendored/urllib3/contrib/pyopenssl.py +509 -0
- snowflake/connector/vendored/urllib3/contrib/securetransport.py +920 -0
- snowflake/connector/vendored/urllib3/contrib/socks.py +216 -0
- snowflake/connector/vendored/urllib3/exceptions.py +323 -0
- snowflake/connector/vendored/urllib3/fields.py +274 -0
- snowflake/connector/vendored/urllib3/filepost.py +98 -0
- snowflake/connector/vendored/urllib3/packages/__init__.py +0 -0
- snowflake/connector/vendored/urllib3/packages/backports/__init__.py +0 -0
- snowflake/connector/vendored/urllib3/packages/backports/makefile.py +51 -0
- snowflake/connector/vendored/urllib3/packages/backports/weakref_finalize.py +155 -0
- snowflake/connector/vendored/urllib3/packages/six.py +1076 -0
- snowflake/connector/vendored/urllib3/poolmanager.py +540 -0
- snowflake/connector/vendored/urllib3/request.py +191 -0
- snowflake/connector/vendored/urllib3/response.py +885 -0
- snowflake/connector/vendored/urllib3/util/__init__.py +49 -0
- snowflake/connector/vendored/urllib3/util/connection.py +156 -0
- snowflake/connector/vendored/urllib3/util/proxy.py +57 -0
- snowflake/connector/vendored/urllib3/util/queue.py +22 -0
- snowflake/connector/vendored/urllib3/util/request.py +146 -0
- snowflake/connector/vendored/urllib3/util/response.py +107 -0
- snowflake/connector/vendored/urllib3/util/retry.py +620 -0
- snowflake/connector/vendored/urllib3/util/ssl_.py +495 -0
- snowflake/connector/vendored/urllib3/util/ssl_match_hostname.py +159 -0
- snowflake/connector/vendored/urllib3/util/ssltransport.py +221 -0
- snowflake/connector/vendored/urllib3/util/timeout.py +271 -0
- snowflake/connector/vendored/urllib3/util/url.py +435 -0
- snowflake/connector/vendored/urllib3/util/wait.py +152 -0
- snowflake/connector/version.py +3 -0
- snowflake/connector/wif_util.py +407 -0
- snowflake_connector_python-3.16.0.dist-info/METADATA +1475 -0
- snowflake_connector_python-3.16.0.dist-info/RECORD +246 -0
- snowflake_connector_python-3.16.0.dist-info/WHEEL +5 -0
- snowflake_connector_python-3.16.0.dist-info/entry_points.txt +4 -0
- snowflake_connector_python-3.16.0.dist-info/licenses/LICENSE.txt +202 -0
- snowflake_connector_python-3.16.0.dist-info/licenses/NOTICE +8 -0
- snowflake_connector_python-3.16.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
#ifndef PC_ARROWTABLEITERATOR_HPP
|
|
2
|
+
#define PC_ARROWTABLEITERATOR_HPP
|
|
3
|
+
|
|
4
|
+
#include <memory>
|
|
5
|
+
#include <string>
|
|
6
|
+
#include <vector>
|
|
7
|
+
|
|
8
|
+
#include "CArrowIterator.hpp"
|
|
9
|
+
#include "nanoarrow.h"
|
|
10
|
+
#include "nanoarrow.hpp"
|
|
11
|
+
|
|
12
|
+
namespace sf {
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Arrow table iterator implementation in C++.
|
|
16
|
+
* The caller will ask for an Arrow Table to be returned back to Python
|
|
17
|
+
* This conversion is zero-copy, just aggregate every columns from multiple
|
|
18
|
+
* record batches and build a new table.
|
|
19
|
+
*/
|
|
20
|
+
class CArrowTableIterator : public CArrowIterator {
|
|
21
|
+
public:
|
|
22
|
+
/**
|
|
23
|
+
* Constructor
|
|
24
|
+
*/
|
|
25
|
+
CArrowTableIterator(PyObject* context, char* arrow_bytes,
|
|
26
|
+
int64_t arrow_bytes_size, bool number_to_decimal);
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Destructor
|
|
30
|
+
*/
|
|
31
|
+
~CArrowTableIterator() = default;
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* @return an arrow table containing all data in all record batches
|
|
35
|
+
*/
|
|
36
|
+
ReturnVal next() override;
|
|
37
|
+
std::vector<uintptr_t> getArrowArrayPtrs() override;
|
|
38
|
+
std::vector<uintptr_t> getArrowSchemaPtrs() override;
|
|
39
|
+
|
|
40
|
+
private:
|
|
41
|
+
// nanoarrow data
|
|
42
|
+
std::vector<nanoarrow::UniqueSchema> m_ipcSchemaArrayVec;
|
|
43
|
+
|
|
44
|
+
bool m_tableConverted = false;
|
|
45
|
+
|
|
46
|
+
/** arrow format convert context for the current session */
|
|
47
|
+
PyObject* m_context;
|
|
48
|
+
|
|
49
|
+
/** local time zone */
|
|
50
|
+
char* m_timezone;
|
|
51
|
+
const bool m_convert_number_to_decimal;
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Reconstruct record batches with type conversion in place
|
|
55
|
+
*/
|
|
56
|
+
void reconstructRecordBatches();
|
|
57
|
+
void convertIfNeeded(ArrowSchema* columnSchema, ArrowArrayView* columnArray);
|
|
58
|
+
void reconstructRecordBatches_nanoarrow();
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Convert all current RecordBatches to Arrow Table
|
|
62
|
+
* @return if conversion is executed at first time and successfully
|
|
63
|
+
*/
|
|
64
|
+
bool convertRecordBatchesToTable_nanoarrow();
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* convert scaled fixed number column to Decimal, or Double column based on
|
|
68
|
+
* setting
|
|
69
|
+
*/
|
|
70
|
+
void convertScaledFixedNumberColumn_nanoarrow(ArrowSchemaView* field,
|
|
71
|
+
ArrowArrayView* columnArray,
|
|
72
|
+
const unsigned int scale);
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* convert scaled fixed number column to Decimal column
|
|
76
|
+
*/
|
|
77
|
+
void convertScaledFixedNumberColumnToDecimalColumn_nanoarrow(
|
|
78
|
+
ArrowSchemaView* field, ArrowArrayView* columnArray,
|
|
79
|
+
const unsigned int scale);
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* convert scaled fixed number column to Double column
|
|
83
|
+
*/
|
|
84
|
+
void convertScaledFixedNumberColumnToDoubleColumn_nanoarrow(
|
|
85
|
+
ArrowSchemaView* field, ArrowArrayView* columnArray,
|
|
86
|
+
const unsigned int scale);
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* convert Snowflake Time column (Arrow int32/int64) to Arrow Time column
|
|
90
|
+
* Since Python/Pandas Time does not support nanoseconds, this function
|
|
91
|
+
* truncates values to microseconds if necessary
|
|
92
|
+
*/
|
|
93
|
+
void convertTimeColumn_nanoarrow(ArrowSchemaView* field,
|
|
94
|
+
ArrowArrayView* columnArray,
|
|
95
|
+
const int scale);
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* convert Snowflake TimestampNTZ/TimestampLTZ column to Arrow Timestamp
|
|
99
|
+
* column
|
|
100
|
+
*/
|
|
101
|
+
void convertTimestampColumn_nanoarrow(ArrowSchemaView* field,
|
|
102
|
+
ArrowArrayView* columnArray,
|
|
103
|
+
const int scale,
|
|
104
|
+
const std::string timezone = "");
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* convert Snowflake TimestampTZ column to Arrow Timestamp column in UTC
|
|
108
|
+
* Arrow Timestamp does not support time zone info in each value, so this
|
|
109
|
+
* method convert TimestampTZ to Arrow timestamp with UTC timezone
|
|
110
|
+
*/
|
|
111
|
+
void convertTimestampTZColumn_nanoarrow(ArrowSchemaView* field,
|
|
112
|
+
ArrowArrayView* columnArray,
|
|
113
|
+
const int scale, const int byteLength,
|
|
114
|
+
const std::string timezone);
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* convert scaled fixed number to double
|
|
118
|
+
* if scale is small, then just divide based on the scale; otherwise, convert
|
|
119
|
+
* the value to string first and then convert to double to avoid precision
|
|
120
|
+
* loss
|
|
121
|
+
*/
|
|
122
|
+
template <typename T>
|
|
123
|
+
double convertScaledFixedNumberToDouble(const unsigned int scale,
|
|
124
|
+
T originalValue);
|
|
125
|
+
};
|
|
126
|
+
} // namespace sf
|
|
127
|
+
#endif // PC_ARROWTABLEITERATOR_HPP
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
#include "DateConverter.hpp"
|
|
2
|
+
|
|
3
|
+
#include <memory>
|
|
4
|
+
|
|
5
|
+
#include "Python/Helpers.hpp"
|
|
6
|
+
|
|
7
|
+
namespace sf {
|
|
8
|
+
Logger* DateConverter::logger = new Logger("snowflake.connector.DateConverter");
|
|
9
|
+
|
|
10
|
+
py::UniqueRef& DateConverter::initPyDatetimeDate() {
|
|
11
|
+
static py::UniqueRef pyDatetimeDate;
|
|
12
|
+
if (pyDatetimeDate.empty()) {
|
|
13
|
+
py::UniqueRef pyDatetimeModule;
|
|
14
|
+
py::importPythonModule("datetime", pyDatetimeModule);
|
|
15
|
+
py::importFromModule(pyDatetimeModule, "date", pyDatetimeDate);
|
|
16
|
+
Py_XINCREF(pyDatetimeDate.get());
|
|
17
|
+
}
|
|
18
|
+
return pyDatetimeDate;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
DateConverter::DateConverter(ArrowArrayView* array)
|
|
22
|
+
: m_array(array), m_pyDatetimeDate(initPyDatetimeDate()) {}
|
|
23
|
+
|
|
24
|
+
PyObject* DateConverter::toPyObject(int64_t rowIndex) const {
|
|
25
|
+
if (ArrowArrayViewIsNull(m_array, rowIndex)) {
|
|
26
|
+
Py_RETURN_NONE;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
int64_t deltaDays = ArrowArrayViewGetIntUnsafe(m_array, rowIndex);
|
|
30
|
+
return PyObject_CallMethod(m_pyDatetimeDate.get(), "fromordinal", "i",
|
|
31
|
+
epochDay + deltaDays);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
NumpyDateConverter::NumpyDateConverter(ArrowArrayView* array, PyObject* context)
|
|
35
|
+
: m_array(array), m_context(context) {}
|
|
36
|
+
|
|
37
|
+
PyObject* NumpyDateConverter::toPyObject(int64_t rowIndex) const {
|
|
38
|
+
if (ArrowArrayViewIsNull(m_array, rowIndex)) {
|
|
39
|
+
Py_RETURN_NONE;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
int64_t deltaDays = ArrowArrayViewGetIntUnsafe(m_array, rowIndex);
|
|
43
|
+
return PyObject_CallMethod(m_context, "DATE_to_numpy_datetime64", "i",
|
|
44
|
+
deltaDays);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
} // namespace sf
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
#ifndef PC_DATECONVERTER_HPP
|
|
2
|
+
#define PC_DATECONVERTER_HPP
|
|
3
|
+
|
|
4
|
+
#include <memory>
|
|
5
|
+
|
|
6
|
+
#include "IColumnConverter.hpp"
|
|
7
|
+
#include "Python/Common.hpp"
|
|
8
|
+
#include "logging.hpp"
|
|
9
|
+
#include "nanoarrow.h"
|
|
10
|
+
|
|
11
|
+
namespace sf {
|
|
12
|
+
|
|
13
|
+
class DateConverter : public IColumnConverter {
|
|
14
|
+
public:
|
|
15
|
+
explicit DateConverter(ArrowArrayView* array);
|
|
16
|
+
|
|
17
|
+
PyObject* toPyObject(int64_t rowIndex) const override;
|
|
18
|
+
|
|
19
|
+
private:
|
|
20
|
+
static py::UniqueRef& initPyDatetimeDate();
|
|
21
|
+
|
|
22
|
+
ArrowArrayView* m_array;
|
|
23
|
+
|
|
24
|
+
/** from Python Ordinal to 1970-01-01 */
|
|
25
|
+
static constexpr int epochDay = 719163;
|
|
26
|
+
|
|
27
|
+
static Logger* logger;
|
|
28
|
+
|
|
29
|
+
py::UniqueRef& m_pyDatetimeDate;
|
|
30
|
+
};
|
|
31
|
+
|
|
32
|
+
class NumpyDateConverter : public IColumnConverter {
|
|
33
|
+
public:
|
|
34
|
+
explicit NumpyDateConverter(ArrowArrayView* array, PyObject* context);
|
|
35
|
+
|
|
36
|
+
PyObject* toPyObject(int64_t rowIndex) const override;
|
|
37
|
+
|
|
38
|
+
private:
|
|
39
|
+
ArrowArrayView* m_array;
|
|
40
|
+
|
|
41
|
+
PyObject* m_context;
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
} // namespace sf
|
|
45
|
+
|
|
46
|
+
#endif // PC_DATECONVERTER_HPP
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
|
|
2
|
+
#include "DecFloatConverter.hpp"
|
|
3
|
+
|
|
4
|
+
#include <cstring>
|
|
5
|
+
#include <memory>
|
|
6
|
+
|
|
7
|
+
#include "Python/Helpers.hpp"
|
|
8
|
+
|
|
9
|
+
namespace sf {
|
|
10
|
+
|
|
11
|
+
Logger* DecFloatConverter::logger =
|
|
12
|
+
new Logger("snowflake.connector.DecFloatConverter");
|
|
13
|
+
|
|
14
|
+
const std::string DecFloatConverter::FIELD_NAME_EXPONENT = "exponent";
|
|
15
|
+
const std::string DecFloatConverter::FIELD_NAME_SIGNIFICAND = "significand";
|
|
16
|
+
|
|
17
|
+
DecFloatConverter::DecFloatConverter(ArrowArrayView& array,
|
|
18
|
+
ArrowSchemaView& schema, PyObject& context,
|
|
19
|
+
bool useNumpy)
|
|
20
|
+
: m_context(context),
|
|
21
|
+
m_array(array),
|
|
22
|
+
m_exponent(nullptr),
|
|
23
|
+
m_significand(nullptr),
|
|
24
|
+
m_useNumpy(useNumpy) {
|
|
25
|
+
if (schema.schema->n_children != 2) {
|
|
26
|
+
std::string errorInfo = Logger::formatString(
|
|
27
|
+
"[Snowflake Exception] arrow schema field number does not match, "
|
|
28
|
+
"expected 2 but got %d instead",
|
|
29
|
+
schema.schema->n_children);
|
|
30
|
+
logger->error(__FILE__, __func__, __LINE__, errorInfo.c_str());
|
|
31
|
+
PyErr_SetString(PyExc_Exception, errorInfo.c_str());
|
|
32
|
+
return;
|
|
33
|
+
}
|
|
34
|
+
for (int i = 0; i < schema.schema->n_children; i += 1) {
|
|
35
|
+
ArrowSchema* c_schema = schema.schema->children[i];
|
|
36
|
+
if (std::strcmp(c_schema->name,
|
|
37
|
+
DecFloatConverter::FIELD_NAME_EXPONENT.c_str()) == 0) {
|
|
38
|
+
m_exponent = m_array.children[i];
|
|
39
|
+
} else if (std::strcmp(c_schema->name,
|
|
40
|
+
DecFloatConverter::FIELD_NAME_SIGNIFICAND.c_str()) ==
|
|
41
|
+
0) {
|
|
42
|
+
m_significand = m_array.children[i];
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
if (!m_exponent || !m_significand) {
|
|
46
|
+
std::string errorInfo = Logger::formatString(
|
|
47
|
+
"[Snowflake Exception] arrow schema field names do not match, "
|
|
48
|
+
"expected %s and %s, but got %s and %s instead",
|
|
49
|
+
DecFloatConverter::FIELD_NAME_EXPONENT.c_str(),
|
|
50
|
+
DecFloatConverter::FIELD_NAME_SIGNIFICAND.c_str(),
|
|
51
|
+
schema.schema->children[0]->name, schema.schema->children[1]->name);
|
|
52
|
+
logger->error(__FILE__, __func__, __LINE__, errorInfo.c_str());
|
|
53
|
+
PyErr_SetString(PyExc_Exception, errorInfo.c_str());
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
PyObject* DecFloatConverter::toPyObject(int64_t rowIndex) const {
|
|
59
|
+
if (ArrowArrayViewIsNull(&m_array, rowIndex)) {
|
|
60
|
+
Py_RETURN_NONE;
|
|
61
|
+
}
|
|
62
|
+
int64_t exponent = ArrowArrayViewGetIntUnsafe(m_exponent, rowIndex);
|
|
63
|
+
ArrowStringView stringView =
|
|
64
|
+
ArrowArrayViewGetStringUnsafe(m_significand, rowIndex);
|
|
65
|
+
if (stringView.size_bytes > 16) {
|
|
66
|
+
std::string errorInfo = Logger::formatString(
|
|
67
|
+
"[Snowflake Exception] only precisions up to 38 supported. "
|
|
68
|
+
"Please update to a newer version of the connector.");
|
|
69
|
+
logger->error(__FILE__, __func__, __LINE__, errorInfo.c_str());
|
|
70
|
+
PyErr_SetString(PyExc_Exception, errorInfo.c_str());
|
|
71
|
+
return nullptr;
|
|
72
|
+
}
|
|
73
|
+
PyObject* significand =
|
|
74
|
+
PyBytes_FromStringAndSize(stringView.data, stringView.size_bytes);
|
|
75
|
+
|
|
76
|
+
PyObject* result = PyObject_CallMethod(
|
|
77
|
+
&m_context,
|
|
78
|
+
m_useNumpy ? "DECFLOAT_to_numpy_float64" : "DECFLOAT_to_decimal", "iS",
|
|
79
|
+
exponent, significand);
|
|
80
|
+
Py_XDECREF(significand);
|
|
81
|
+
return result;
|
|
82
|
+
}
|
|
83
|
+
} // namespace sf
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
|
|
2
|
+
#ifndef PC_DECFLOATCONVERTER_HPP
|
|
3
|
+
#define PC_DECFLOATCONVERTER_HPP
|
|
4
|
+
|
|
5
|
+
#include <memory>
|
|
6
|
+
|
|
7
|
+
#include "IColumnConverter.hpp"
|
|
8
|
+
#include "logging.hpp"
|
|
9
|
+
#include "nanoarrow.h"
|
|
10
|
+
|
|
11
|
+
namespace sf {
|
|
12
|
+
|
|
13
|
+
class DecFloatConverter : public IColumnConverter {
|
|
14
|
+
public:
|
|
15
|
+
const static std::string FIELD_NAME_EXPONENT;
|
|
16
|
+
const static std::string FIELD_NAME_SIGNIFICAND;
|
|
17
|
+
|
|
18
|
+
explicit DecFloatConverter(ArrowArrayView& array, ArrowSchemaView& schema,
|
|
19
|
+
PyObject& context, bool useNumpy);
|
|
20
|
+
|
|
21
|
+
PyObject* toPyObject(int64_t rowIndex) const override;
|
|
22
|
+
|
|
23
|
+
private:
|
|
24
|
+
PyObject& m_context;
|
|
25
|
+
ArrowArrayView& m_array;
|
|
26
|
+
ArrowArrayView* m_exponent;
|
|
27
|
+
ArrowArrayView* m_significand;
|
|
28
|
+
bool m_useNumpy;
|
|
29
|
+
|
|
30
|
+
static Logger* logger;
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
} // namespace sf
|
|
34
|
+
|
|
35
|
+
#endif // PC_DECFLOATCONVERTER_HPP
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
#include "DecimalConverter.hpp"
|
|
2
|
+
|
|
3
|
+
#include <memory>
|
|
4
|
+
#include <string>
|
|
5
|
+
|
|
6
|
+
#include "Python/Common.hpp"
|
|
7
|
+
#include "Python/Helpers.hpp"
|
|
8
|
+
|
|
9
|
+
namespace sf {
|
|
10
|
+
|
|
11
|
+
DecimalBaseConverter::DecimalBaseConverter()
|
|
12
|
+
: m_pyDecimalConstructor(initPyDecimalConstructor()) {}
|
|
13
|
+
|
|
14
|
+
py::UniqueRef& DecimalBaseConverter::initPyDecimalConstructor() {
|
|
15
|
+
static py::UniqueRef pyDecimalConstructor;
|
|
16
|
+
if (pyDecimalConstructor.empty()) {
|
|
17
|
+
py::UniqueRef decimalModule;
|
|
18
|
+
py::importPythonModule("decimal", decimalModule);
|
|
19
|
+
py::importFromModule(decimalModule, "Decimal", pyDecimalConstructor);
|
|
20
|
+
Py_XINCREF(pyDecimalConstructor.get());
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
return pyDecimalConstructor;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
DecimalFromIntConverter::DecimalFromIntConverter(ArrowArrayView* array,
|
|
27
|
+
int precision, int scale)
|
|
28
|
+
: m_array(array), m_precision(precision), m_scale(scale) {}
|
|
29
|
+
|
|
30
|
+
PyObject* DecimalFromIntConverter::toPyObject(int64_t rowIndex) const {
|
|
31
|
+
if (ArrowArrayViewIsNull(m_array, rowIndex)) {
|
|
32
|
+
Py_RETURN_NONE;
|
|
33
|
+
}
|
|
34
|
+
int64_t val = ArrowArrayViewGetIntUnsafe(m_array, rowIndex);
|
|
35
|
+
py::UniqueRef decimal(
|
|
36
|
+
PyObject_CallFunction(m_pyDecimalConstructor.get(), "L", val));
|
|
37
|
+
return PyObject_CallMethod(decimal.get(), "scaleb", "i", -m_scale);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
NumpyDecimalConverter::NumpyDecimalConverter(ArrowArrayView* array,
|
|
41
|
+
int precision, int scale,
|
|
42
|
+
PyObject* context)
|
|
43
|
+
: m_array(array),
|
|
44
|
+
m_precision(precision),
|
|
45
|
+
m_scale(scale),
|
|
46
|
+
m_context(context) {}
|
|
47
|
+
|
|
48
|
+
PyObject* NumpyDecimalConverter::toPyObject(int64_t rowIndex) const {
|
|
49
|
+
if (ArrowArrayViewIsNull(m_array, rowIndex)) {
|
|
50
|
+
Py_RETURN_NONE;
|
|
51
|
+
}
|
|
52
|
+
int64_t val = ArrowArrayViewGetIntUnsafe(m_array, rowIndex);
|
|
53
|
+
return PyObject_CallMethod(m_context, "FIXED_to_numpy_float64", "Li", val,
|
|
54
|
+
m_scale);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
DecimalFromDecimalConverter::DecimalFromDecimalConverter(PyObject* context,
|
|
58
|
+
ArrowArrayView* array,
|
|
59
|
+
int scale)
|
|
60
|
+
: m_array(array), m_context(context), m_scale(scale) {}
|
|
61
|
+
|
|
62
|
+
PyObject* DecimalFromDecimalConverter::toPyObject(int64_t rowIndex) const {
|
|
63
|
+
if (ArrowArrayViewIsNull(m_array, rowIndex)) {
|
|
64
|
+
Py_RETURN_NONE;
|
|
65
|
+
}
|
|
66
|
+
int64_t bytes_start = 16 * (m_array->array->offset + rowIndex);
|
|
67
|
+
const char* ptr_start = m_array->buffer_views[1].data.as_char;
|
|
68
|
+
PyObject* int128_bytes =
|
|
69
|
+
PyBytes_FromStringAndSize(&(ptr_start[bytes_start]), 16);
|
|
70
|
+
/**
|
|
71
|
+
# Alternatively, the decimal conversion can be implemented using the
|
|
72
|
+
ArrowDecimal related APIs in the following # code snippets, however, it's less
|
|
73
|
+
performant than the direct memory manipulation. # The code snippets here is
|
|
74
|
+
for context reference.
|
|
75
|
+
|
|
76
|
+
ArrowDecimal arrowDecimal;
|
|
77
|
+
ArrowDecimalInit(&arrowDecimal, 128, precision, scale);
|
|
78
|
+
ArrowArrayViewGetDecimalUnsafe(m_array, rowIndex, &arrowDecimal);
|
|
79
|
+
uint8_t outBytes[16];
|
|
80
|
+
ArrowDecimalGetBytes(&arrowDecimal, outBytes);
|
|
81
|
+
PyObject* int128_bytes = PyBytes_FromStringAndSize(&outBytes, 16);
|
|
82
|
+
*/
|
|
83
|
+
PyObject* return_object = PyObject_CallMethod(
|
|
84
|
+
m_context, "DECIMAL128_to_decimal", "Si", int128_bytes, m_scale);
|
|
85
|
+
/**
|
|
86
|
+
int128_bytes is a new referenced created by PyBytes_FromStringAndSize,
|
|
87
|
+
to avoid memory leak we need to free it after usage
|
|
88
|
+
check docs:
|
|
89
|
+
https://docs.python.org/3/c-api/bytes.html#c.PyBytes_FromStringAndSize
|
|
90
|
+
https://docs.python.org/3/c-api/refcounting.html#c.Py_XDECREF
|
|
91
|
+
|
|
92
|
+
*/
|
|
93
|
+
Py_XDECREF(int128_bytes);
|
|
94
|
+
return return_object;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
} // namespace sf
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
#ifndef PC_DECIMALCONVERTER_HPP
|
|
2
|
+
#define PC_DECIMALCONVERTER_HPP
|
|
3
|
+
|
|
4
|
+
#include <memory>
|
|
5
|
+
|
|
6
|
+
#include "IColumnConverter.hpp"
|
|
7
|
+
#include "Python/Common.hpp"
|
|
8
|
+
#include "nanoarrow.h"
|
|
9
|
+
|
|
10
|
+
namespace sf {
|
|
11
|
+
|
|
12
|
+
class DecimalBaseConverter : public IColumnConverter {
|
|
13
|
+
public:
|
|
14
|
+
DecimalBaseConverter();
|
|
15
|
+
virtual ~DecimalBaseConverter() = default;
|
|
16
|
+
|
|
17
|
+
protected:
|
|
18
|
+
py::UniqueRef& m_pyDecimalConstructor;
|
|
19
|
+
|
|
20
|
+
private:
|
|
21
|
+
static py::UniqueRef& initPyDecimalConstructor();
|
|
22
|
+
};
|
|
23
|
+
|
|
24
|
+
class DecimalFromDecimalConverter : public DecimalBaseConverter {
|
|
25
|
+
public:
|
|
26
|
+
explicit DecimalFromDecimalConverter(PyObject* context, ArrowArrayView* array,
|
|
27
|
+
int scale);
|
|
28
|
+
|
|
29
|
+
PyObject* toPyObject(int64_t rowIndex) const override;
|
|
30
|
+
|
|
31
|
+
private:
|
|
32
|
+
ArrowArrayView* m_array;
|
|
33
|
+
PyObject* m_context;
|
|
34
|
+
int m_scale;
|
|
35
|
+
/** no need for this converter to store precision*/
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
class DecimalFromIntConverter : public DecimalBaseConverter {
|
|
39
|
+
public:
|
|
40
|
+
explicit DecimalFromIntConverter(ArrowArrayView* array, int precision,
|
|
41
|
+
int scale);
|
|
42
|
+
|
|
43
|
+
PyObject* toPyObject(int64_t rowIndex) const override;
|
|
44
|
+
|
|
45
|
+
private:
|
|
46
|
+
ArrowArrayView* m_array;
|
|
47
|
+
int m_precision; // looks like the precision here is not useful, and this
|
|
48
|
+
// will be removed soon when it's been confirmed
|
|
49
|
+
|
|
50
|
+
int m_scale;
|
|
51
|
+
};
|
|
52
|
+
|
|
53
|
+
class NumpyDecimalConverter : public IColumnConverter {
|
|
54
|
+
public:
|
|
55
|
+
explicit NumpyDecimalConverter(ArrowArrayView* array, int precision,
|
|
56
|
+
int scale, PyObject* context);
|
|
57
|
+
|
|
58
|
+
PyObject* toPyObject(int64_t rowIndex) const override;
|
|
59
|
+
|
|
60
|
+
private:
|
|
61
|
+
ArrowArrayView* m_array;
|
|
62
|
+
|
|
63
|
+
int m_precision;
|
|
64
|
+
|
|
65
|
+
int m_scale;
|
|
66
|
+
|
|
67
|
+
PyObject* m_context;
|
|
68
|
+
};
|
|
69
|
+
|
|
70
|
+
} // namespace sf
|
|
71
|
+
|
|
72
|
+
#endif // PC_DECIMALCONVERTER_HPP
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
#include "FixedSizeListConverter.hpp"
|
|
2
|
+
|
|
3
|
+
namespace sf {
|
|
4
|
+
Logger* FixedSizeListConverter::logger =
|
|
5
|
+
new Logger("snowflake.connector.FixedSizeListConverter");
|
|
6
|
+
|
|
7
|
+
FixedSizeListConverter::FixedSizeListConverter(ArrowArrayView* array)
|
|
8
|
+
: m_array(array) {}
|
|
9
|
+
|
|
10
|
+
void FixedSizeListConverter::generateError(const std::string& msg) const {
|
|
11
|
+
logger->error(__FILE__, __func__, __LINE__, msg.c_str());
|
|
12
|
+
PyErr_SetString(PyExc_Exception, msg.c_str());
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
PyObject* FixedSizeListConverter::toPyObject(int64_t rowIndex) const {
|
|
16
|
+
if (ArrowArrayViewIsNull(m_array, rowIndex)) {
|
|
17
|
+
Py_RETURN_NONE;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
if (m_array->n_children != 1) {
|
|
21
|
+
std::string errorInfo = Logger::formatString(
|
|
22
|
+
"[Snowflake Exception] invalid arrow element schema for fixed size "
|
|
23
|
+
"list: got (%d) "
|
|
24
|
+
"children",
|
|
25
|
+
m_array->n_children);
|
|
26
|
+
this->generateError(errorInfo);
|
|
27
|
+
return nullptr;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
// m_array->length represents the number of fixed size lists in the array
|
|
31
|
+
// m_array->children[0] has a buffer view that contains the actual data of
|
|
32
|
+
// each list, back-to-back m_array->children[0]->length represents the sum of
|
|
33
|
+
// the lengths of the fixed size lists in the array.
|
|
34
|
+
|
|
35
|
+
ArrowArrayView* elements = m_array->children[0];
|
|
36
|
+
const auto fixedSizeArrayLength = elements->length / m_array->length;
|
|
37
|
+
PyObject* list = PyList_New(fixedSizeArrayLength);
|
|
38
|
+
|
|
39
|
+
const int64_t startIndexWithoutOffset = rowIndex * fixedSizeArrayLength;
|
|
40
|
+
for (int64_t i = 0; i < fixedSizeArrayLength; ++i) {
|
|
41
|
+
const auto bufferIndexWithoutOffset = startIndexWithoutOffset + i;
|
|
42
|
+
// Currently, the backend only sends back INT32 and FLOAT32, but the
|
|
43
|
+
// remaining types are enumerated for future use.
|
|
44
|
+
switch (elements->storage_type) {
|
|
45
|
+
case NANOARROW_TYPE_INT8:
|
|
46
|
+
case NANOARROW_TYPE_INT16:
|
|
47
|
+
case NANOARROW_TYPE_INT32:
|
|
48
|
+
case NANOARROW_TYPE_INT64: {
|
|
49
|
+
const auto value =
|
|
50
|
+
ArrowArrayViewGetIntUnsafe(elements, bufferIndexWithoutOffset);
|
|
51
|
+
PyList_SetItem(list, i, PyLong_FromLongLong(value));
|
|
52
|
+
} break;
|
|
53
|
+
case NANOARROW_TYPE_HALF_FLOAT:
|
|
54
|
+
case NANOARROW_TYPE_FLOAT:
|
|
55
|
+
case NANOARROW_TYPE_DOUBLE: {
|
|
56
|
+
const auto value =
|
|
57
|
+
ArrowArrayViewGetDoubleUnsafe(elements, bufferIndexWithoutOffset);
|
|
58
|
+
PyList_SetItem(list, i, PyFloat_FromDouble(value));
|
|
59
|
+
} break;
|
|
60
|
+
default:
|
|
61
|
+
std::string errorInfo = Logger::formatString(
|
|
62
|
+
"[Snowflake Exception] invalid arrow element type for fixed size "
|
|
63
|
+
"list: got (%s)",
|
|
64
|
+
ArrowTypeString(elements->storage_type));
|
|
65
|
+
this->generateError(errorInfo);
|
|
66
|
+
return nullptr;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
return list;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
} // namespace sf
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
#ifndef PC_FIXEDSIZELISTCONVERTER_HPP
|
|
2
|
+
#define PC_FIXEDSIZELISTCONVERTER_HPP
|
|
3
|
+
|
|
4
|
+
#include <memory>
|
|
5
|
+
|
|
6
|
+
#include "IColumnConverter.hpp"
|
|
7
|
+
#include "logging.hpp"
|
|
8
|
+
#include "nanoarrow.h"
|
|
9
|
+
#include "nanoarrow.hpp"
|
|
10
|
+
|
|
11
|
+
namespace sf {
|
|
12
|
+
|
|
13
|
+
class FixedSizeListConverter : public IColumnConverter {
|
|
14
|
+
public:
|
|
15
|
+
explicit FixedSizeListConverter(ArrowArrayView* array);
|
|
16
|
+
PyObject* toPyObject(int64_t rowIndex) const override;
|
|
17
|
+
|
|
18
|
+
private:
|
|
19
|
+
void generateError(const std::string& msg) const;
|
|
20
|
+
|
|
21
|
+
ArrowArrayView* m_array;
|
|
22
|
+
|
|
23
|
+
static Logger* logger;
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
} // namespace sf
|
|
27
|
+
|
|
28
|
+
#endif // PC_FIXEDSIZELISTCONVERTER_HPP
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
#include "FloatConverter.hpp"
|
|
2
|
+
|
|
3
|
+
#include <memory>
|
|
4
|
+
|
|
5
|
+
namespace sf {
|
|
6
|
+
|
|
7
|
+
/** snowflake float is 64-precision, which refers to double here */
|
|
8
|
+
FloatConverter::FloatConverter(ArrowArrayView* array) : m_array(array) {}
|
|
9
|
+
|
|
10
|
+
PyObject* FloatConverter::toPyObject(int64_t rowIndex) const {
|
|
11
|
+
if (ArrowArrayViewIsNull(m_array, rowIndex)) {
|
|
12
|
+
Py_RETURN_NONE;
|
|
13
|
+
}
|
|
14
|
+
return PyFloat_FromDouble(ArrowArrayViewGetDoubleUnsafe(m_array, rowIndex));
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
NumpyFloat64Converter::NumpyFloat64Converter(ArrowArrayView* array,
|
|
18
|
+
PyObject* context)
|
|
19
|
+
: m_array(array), m_context(context) {}
|
|
20
|
+
|
|
21
|
+
PyObject* NumpyFloat64Converter::toPyObject(int64_t rowIndex) const {
|
|
22
|
+
if (ArrowArrayViewIsNull(m_array, rowIndex)) {
|
|
23
|
+
Py_RETURN_NONE;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
double val = ArrowArrayViewGetDoubleUnsafe(m_array, rowIndex);
|
|
27
|
+
return PyObject_CallMethod(m_context, "REAL_to_numpy_float64", "d", val);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
} // namespace sf
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
#ifndef PC_FLOATCONVERTER_HPP
|
|
2
|
+
#define PC_FLOATCONVERTER_HPP
|
|
3
|
+
|
|
4
|
+
#include <memory>
|
|
5
|
+
|
|
6
|
+
#include "IColumnConverter.hpp"
|
|
7
|
+
#include "nanoarrow.h"
|
|
8
|
+
|
|
9
|
+
namespace sf {
|
|
10
|
+
|
|
11
|
+
class FloatConverter : public IColumnConverter {
|
|
12
|
+
public:
|
|
13
|
+
explicit FloatConverter(ArrowArrayView* array);
|
|
14
|
+
|
|
15
|
+
PyObject* toPyObject(int64_t rowIndex) const override;
|
|
16
|
+
|
|
17
|
+
private:
|
|
18
|
+
ArrowArrayView* m_array;
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
class NumpyFloat64Converter : public IColumnConverter {
|
|
22
|
+
public:
|
|
23
|
+
explicit NumpyFloat64Converter(ArrowArrayView* array, PyObject* context);
|
|
24
|
+
|
|
25
|
+
PyObject* toPyObject(int64_t rowIndex) const override;
|
|
26
|
+
|
|
27
|
+
private:
|
|
28
|
+
ArrowArrayView* m_array;
|
|
29
|
+
|
|
30
|
+
PyObject* m_context;
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
} // namespace sf
|
|
34
|
+
|
|
35
|
+
#endif // PC_FLOATCONVERTER_HPP
|