snowpark-connect 0.21.0__py3-none-any.whl → 0.22.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of snowpark-connect might be problematic. Click here for more details.
- snowflake/snowpark_connect/config.py +19 -3
- snowflake/snowpark_connect/error/error_utils.py +25 -0
- snowflake/snowpark_connect/expression/map_udf.py +4 -4
- snowflake/snowpark_connect/expression/map_unresolved_function.py +203 -128
- snowflake/snowpark_connect/proto/snowflake_expression_ext_pb2_grpc.py +4 -0
- snowflake/snowpark_connect/proto/snowflake_relation_ext_pb2_grpc.py +4 -0
- snowflake/snowpark_connect/relation/map_aggregate.py +102 -18
- snowflake/snowpark_connect/relation/map_column_ops.py +21 -2
- snowflake/snowpark_connect/relation/map_map_partitions.py +3 -1
- snowflake/snowpark_connect/relation/map_sql.py +18 -191
- snowflake/snowpark_connect/relation/map_udtf.py +4 -4
- snowflake/snowpark_connect/relation/read/map_read_json.py +12 -1
- snowflake/snowpark_connect/relation/read/reader_config.py +1 -0
- snowflake/snowpark_connect/relation/write/map_write.py +68 -24
- snowflake/snowpark_connect/server.py +9 -0
- snowflake/snowpark_connect/type_mapping.py +4 -0
- snowflake/snowpark_connect/utils/describe_query_cache.py +2 -9
- snowflake/snowpark_connect/utils/session.py +0 -4
- snowflake/snowpark_connect/utils/telemetry.py +213 -61
- snowflake/snowpark_connect/utils/udxf_import_utils.py +14 -0
- snowflake/snowpark_connect/version.py +1 -1
- snowflake/snowpark_decoder/__init__.py +0 -0
- snowflake/snowpark_decoder/_internal/proto/generated/DataframeProcessorMsg_pb2.py +36 -0
- snowflake/snowpark_decoder/_internal/proto/generated/DataframeProcessorMsg_pb2.pyi +156 -0
- snowflake/snowpark_decoder/dp_session.py +111 -0
- snowflake/snowpark_decoder/spark_decoder.py +76 -0
- {snowpark_connect-0.21.0.dist-info → snowpark_connect-0.22.1.dist-info}/METADATA +2 -2
- {snowpark_connect-0.21.0.dist-info → snowpark_connect-0.22.1.dist-info}/RECORD +40 -29
- {snowpark_connect-0.21.0.dist-info → snowpark_connect-0.22.1.dist-info}/top_level.txt +1 -0
- spark/__init__.py +0 -0
- spark/connect/__init__.py +0 -0
- spark/connect/envelope_pb2.py +31 -0
- spark/connect/envelope_pb2.pyi +46 -0
- snowflake/snowpark_connect/includes/jars/jackson-mapper-asl-1.9.13.jar +0 -0
- {snowpark_connect-0.21.0.data → snowpark_connect-0.22.1.data}/scripts/snowpark-connect +0 -0
- {snowpark_connect-0.21.0.data → snowpark_connect-0.22.1.data}/scripts/snowpark-session +0 -0
- {snowpark_connect-0.21.0.data → snowpark_connect-0.22.1.data}/scripts/snowpark-submit +0 -0
- {snowpark_connect-0.21.0.dist-info → snowpark_connect-0.22.1.dist-info}/WHEEL +0 -0
- {snowpark_connect-0.21.0.dist-info → snowpark_connect-0.22.1.dist-info}/licenses/LICENSE-binary +0 -0
- {snowpark_connect-0.21.0.dist-info → snowpark_connect-0.22.1.dist-info}/licenses/LICENSE.txt +0 -0
- {snowpark_connect-0.21.0.dist-info → snowpark_connect-0.22.1.dist-info}/licenses/NOTICE-binary +0 -0
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Copyright (c) 2012-2025 Snowflake Computing Inc. All rights reserved.
|
|
3
|
+
#
|
|
4
|
+
|
|
5
|
+
from snowflake import snowpark
|
|
6
|
+
from snowflake.snowpark_connect.config import global_config
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def get_python_udxf_import_files(session: snowpark.Session) -> str:
|
|
10
|
+
config_imports = global_config.get("snowpark.connect.udf.imports", "")
|
|
11
|
+
config_imports = config_imports.strip("[] ").split(",") if config_imports else []
|
|
12
|
+
imports = {*session._python_files, *session._import_files, *config_imports}
|
|
13
|
+
|
|
14
|
+
return ",".join([file for file in imports if file])
|
|
File without changes
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
|
3
|
+
# source: DataframeProcessorMsg.proto
|
|
4
|
+
"""Generated protocol buffer code."""
|
|
5
|
+
from google.protobuf.internal import builder as _builder
|
|
6
|
+
from google.protobuf import descriptor as _descriptor
|
|
7
|
+
from google.protobuf import descriptor_pool as _descriptor_pool
|
|
8
|
+
from google.protobuf import symbol_database as _symbol_database
|
|
9
|
+
# @@protoc_insertion_point(imports)
|
|
10
|
+
|
|
11
|
+
_sym_db = _symbol_database.Default()
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
from google.protobuf import descriptor_pb2 as google_dot_protobuf_dot_descriptor__pb2
|
|
15
|
+
from google.protobuf import any_pb2 as google_dot_protobuf_dot_any__pb2
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1b\x44\x61taframeProcessorMsg.proto\x12\x13\x64\x61taframe_processor\x1a google/protobuf/descriptor.proto\x1a\x19google/protobuf/any.proto\"\x80\x01\n\x07Request\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12:\n\x0e\x64\x61taframe_type\x18\x02 \x01(\x0e\x32\".dataframe_processor.DataframeType\x12%\n\x07payload\x18\x03 \x01(\x0b\x32\x14.google.protobuf.Any\"\xd4\x02\n\x08Response\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12:\n\x0e\x64\x61taframe_type\x18\x02 \x01(\x0e\x32\".dataframe_processor.DataframeType\x12\x36\n\x04\x63ode\x18\x03 \x01(\x0e\x32(.dataframe_processor.Response.StatusCode\x12\x19\n\x0fresult_job_uuid\x18\x04 \x01(\tH\x00\x12\'\n\x07payload\x18\x05 \x01(\x0b\x32\x14.google.protobuf.AnyH\x00\x12\x15\n\rerror_message\x18\x06 \x01(\t\"[\n\nStatusCode\x12\x06\n\x02OK\x10\x00\x12\x0e\n\nINIT_ERROR\x10\x01\x12\x13\n\x0f\x45XECUTION_ERROR\x10\x02\x12 \n\x1cSESSION_RESET_REQUIRED_ERROR\x10\x03\x42\x08\n\x06result\"\x18\n\x16TruncatedSparkRelation*C\n\rDataframeType\x12\r\n\tUNDEFINED\x10\x00\x12\x10\n\x0cSNOWPARK_API\x10\x01\x12\x11\n\rSPARK_CONNECT\x10\x02\x42$\n\x17\x63om.snowflake.dataframeB\x06Protos\x80\x01\x00\x62\x06proto3')
|
|
19
|
+
|
|
20
|
+
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
|
|
21
|
+
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'DataframeProcessorMsg_pb2', globals())
|
|
22
|
+
if _descriptor._USE_C_DESCRIPTORS == False:
|
|
23
|
+
|
|
24
|
+
DESCRIPTOR._options = None
|
|
25
|
+
DESCRIPTOR._serialized_options = b'\n\027com.snowflake.dataframeB\006Protos\200\001\000'
|
|
26
|
+
_DATAFRAMETYPE._serialized_start=613
|
|
27
|
+
_DATAFRAMETYPE._serialized_end=680
|
|
28
|
+
_REQUEST._serialized_start=114
|
|
29
|
+
_REQUEST._serialized_end=242
|
|
30
|
+
_RESPONSE._serialized_start=245
|
|
31
|
+
_RESPONSE._serialized_end=585
|
|
32
|
+
_RESPONSE_STATUSCODE._serialized_start=484
|
|
33
|
+
_RESPONSE_STATUSCODE._serialized_end=575
|
|
34
|
+
_TRUNCATEDSPARKRELATION._serialized_start=587
|
|
35
|
+
_TRUNCATEDSPARKRELATION._serialized_end=611
|
|
36
|
+
# @@protoc_insertion_point(module_scope)
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
"""
|
|
2
|
+
@generated by mypy-protobuf. Do not edit manually!
|
|
3
|
+
isort:skip_file
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
import builtins
|
|
7
|
+
import google.protobuf.any_pb2
|
|
8
|
+
import google.protobuf.descriptor
|
|
9
|
+
import google.protobuf.internal.enum_type_wrapper
|
|
10
|
+
import google.protobuf.message
|
|
11
|
+
import sys
|
|
12
|
+
import typing
|
|
13
|
+
|
|
14
|
+
if sys.version_info >= (3, 10):
|
|
15
|
+
import typing as typing_extensions
|
|
16
|
+
else:
|
|
17
|
+
import typing_extensions
|
|
18
|
+
|
|
19
|
+
DESCRIPTOR: google.protobuf.descriptor.FileDescriptor
|
|
20
|
+
|
|
21
|
+
class _DataframeType:
|
|
22
|
+
ValueType = typing.NewType("ValueType", builtins.int)
|
|
23
|
+
V: typing_extensions.TypeAlias = ValueType
|
|
24
|
+
|
|
25
|
+
class _DataframeTypeEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[_DataframeType.ValueType], builtins.type):
|
|
26
|
+
DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor
|
|
27
|
+
UNDEFINED: _DataframeType.ValueType # 0
|
|
28
|
+
"""The type is undefined"""
|
|
29
|
+
SNOWPARK_API: _DataframeType.ValueType # 1
|
|
30
|
+
"""Snowpark API"""
|
|
31
|
+
SPARK_CONNECT: _DataframeType.ValueType # 2
|
|
32
|
+
"""Spark connect API"""
|
|
33
|
+
|
|
34
|
+
class DataframeType(_DataframeType, metaclass=_DataframeTypeEnumTypeWrapper):
|
|
35
|
+
"""The type of the dataframe"""
|
|
36
|
+
|
|
37
|
+
UNDEFINED: DataframeType.ValueType # 0
|
|
38
|
+
"""The type is undefined"""
|
|
39
|
+
SNOWPARK_API: DataframeType.ValueType # 1
|
|
40
|
+
"""Snowpark API"""
|
|
41
|
+
SPARK_CONNECT: DataframeType.ValueType # 2
|
|
42
|
+
"""Spark connect API"""
|
|
43
|
+
global___DataframeType = DataframeType
|
|
44
|
+
|
|
45
|
+
@typing.final
|
|
46
|
+
class Request(google.protobuf.message.Message):
|
|
47
|
+
"""The dataframe processor request"""
|
|
48
|
+
|
|
49
|
+
DESCRIPTOR: google.protobuf.descriptor.Descriptor
|
|
50
|
+
|
|
51
|
+
REQUEST_ID_FIELD_NUMBER: builtins.int
|
|
52
|
+
DATAFRAME_TYPE_FIELD_NUMBER: builtins.int
|
|
53
|
+
PAYLOAD_FIELD_NUMBER: builtins.int
|
|
54
|
+
request_id: builtins.str
|
|
55
|
+
dataframe_type: global___DataframeType.ValueType
|
|
56
|
+
@property
|
|
57
|
+
def payload(self) -> google.protobuf.any_pb2.Any:
|
|
58
|
+
"""Type-specific payload.
|
|
59
|
+
SNOWPARK_API requests use the protobuf generated from //Snowpark/ast: ast.ir => `entity Request`.
|
|
60
|
+
SPARK_CONNECT requests use the public Spark Connect protocol.
|
|
61
|
+
... future extensions, e.g., telemetry, metrics config.
|
|
62
|
+
"""
|
|
63
|
+
|
|
64
|
+
def __init__(
|
|
65
|
+
self,
|
|
66
|
+
*,
|
|
67
|
+
request_id: builtins.str = ...,
|
|
68
|
+
dataframe_type: global___DataframeType.ValueType = ...,
|
|
69
|
+
payload: google.protobuf.any_pb2.Any | None = ...,
|
|
70
|
+
) -> None: ...
|
|
71
|
+
def HasField(self, field_name: typing.Literal["payload", b"payload"]) -> builtins.bool: ...
|
|
72
|
+
def ClearField(self, field_name: typing.Literal["dataframe_type", b"dataframe_type", "payload", b"payload", "request_id", b"request_id"]) -> None: ...
|
|
73
|
+
|
|
74
|
+
global___Request = Request
|
|
75
|
+
|
|
76
|
+
@typing.final
|
|
77
|
+
class Response(google.protobuf.message.Message):
|
|
78
|
+
"""Response of the dataframe processor request"""
|
|
79
|
+
|
|
80
|
+
DESCRIPTOR: google.protobuf.descriptor.Descriptor
|
|
81
|
+
|
|
82
|
+
class _StatusCode:
|
|
83
|
+
ValueType = typing.NewType("ValueType", builtins.int)
|
|
84
|
+
V: typing_extensions.TypeAlias = ValueType
|
|
85
|
+
|
|
86
|
+
class _StatusCodeEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[Response._StatusCode.ValueType], builtins.type):
|
|
87
|
+
DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor
|
|
88
|
+
OK: Response._StatusCode.ValueType # 0
|
|
89
|
+
INIT_ERROR: Response._StatusCode.ValueType # 1
|
|
90
|
+
"""Failed to initialize"""
|
|
91
|
+
EXECUTION_ERROR: Response._StatusCode.ValueType # 2
|
|
92
|
+
"""Failed to execute the request"""
|
|
93
|
+
SESSION_RESET_REQUIRED_ERROR: Response._StatusCode.ValueType # 3
|
|
94
|
+
"""Session requires to retransmit the dependencies
|
|
95
|
+
... other status codes
|
|
96
|
+
"""
|
|
97
|
+
|
|
98
|
+
class StatusCode(_StatusCode, metaclass=_StatusCodeEnumTypeWrapper):
|
|
99
|
+
"""The types of errors"""
|
|
100
|
+
|
|
101
|
+
OK: Response.StatusCode.ValueType # 0
|
|
102
|
+
INIT_ERROR: Response.StatusCode.ValueType # 1
|
|
103
|
+
"""Failed to initialize"""
|
|
104
|
+
EXECUTION_ERROR: Response.StatusCode.ValueType # 2
|
|
105
|
+
"""Failed to execute the request"""
|
|
106
|
+
SESSION_RESET_REQUIRED_ERROR: Response.StatusCode.ValueType # 3
|
|
107
|
+
"""Session requires to retransmit the dependencies
|
|
108
|
+
... other status codes
|
|
109
|
+
"""
|
|
110
|
+
|
|
111
|
+
REQUEST_ID_FIELD_NUMBER: builtins.int
|
|
112
|
+
DATAFRAME_TYPE_FIELD_NUMBER: builtins.int
|
|
113
|
+
CODE_FIELD_NUMBER: builtins.int
|
|
114
|
+
RESULT_JOB_UUID_FIELD_NUMBER: builtins.int
|
|
115
|
+
PAYLOAD_FIELD_NUMBER: builtins.int
|
|
116
|
+
ERROR_MESSAGE_FIELD_NUMBER: builtins.int
|
|
117
|
+
request_id: builtins.str
|
|
118
|
+
"""ID of the request with which this response is associated."""
|
|
119
|
+
dataframe_type: global___DataframeType.ValueType
|
|
120
|
+
"""The type of the dataframe"""
|
|
121
|
+
code: global___Response.StatusCode.ValueType
|
|
122
|
+
result_job_uuid: builtins.str
|
|
123
|
+
"""The UUID of the job containing the result of the request."""
|
|
124
|
+
error_message: builtins.str
|
|
125
|
+
"""Dataframe processor internal use only. The external protocol status is in the payload."""
|
|
126
|
+
@property
|
|
127
|
+
def payload(self) -> google.protobuf.any_pb2.Any:
|
|
128
|
+
"""The payload of the response for other types of responses."""
|
|
129
|
+
|
|
130
|
+
def __init__(
|
|
131
|
+
self,
|
|
132
|
+
*,
|
|
133
|
+
request_id: builtins.str = ...,
|
|
134
|
+
dataframe_type: global___DataframeType.ValueType = ...,
|
|
135
|
+
code: global___Response.StatusCode.ValueType = ...,
|
|
136
|
+
result_job_uuid: builtins.str = ...,
|
|
137
|
+
payload: google.protobuf.any_pb2.Any | None = ...,
|
|
138
|
+
error_message: builtins.str = ...,
|
|
139
|
+
) -> None: ...
|
|
140
|
+
def HasField(self, field_name: typing.Literal["payload", b"payload", "result", b"result", "result_job_uuid", b"result_job_uuid"]) -> builtins.bool: ...
|
|
141
|
+
def ClearField(self, field_name: typing.Literal["code", b"code", "dataframe_type", b"dataframe_type", "error_message", b"error_message", "payload", b"payload", "request_id", b"request_id", "result", b"result", "result_job_uuid", b"result_job_uuid"]) -> None: ...
|
|
142
|
+
def WhichOneof(self, oneof_group: typing.Literal["result", b"result"]) -> typing.Literal["result_job_uuid", "payload"] | None: ...
|
|
143
|
+
|
|
144
|
+
global___Response = Response
|
|
145
|
+
|
|
146
|
+
@typing.final
|
|
147
|
+
class TruncatedSparkRelation(google.protobuf.message.Message):
|
|
148
|
+
"""* Empty relation indicating that the real spark relation has been truncated *"""
|
|
149
|
+
|
|
150
|
+
DESCRIPTOR: google.protobuf.descriptor.Descriptor
|
|
151
|
+
|
|
152
|
+
def __init__(
|
|
153
|
+
self,
|
|
154
|
+
) -> None: ...
|
|
155
|
+
|
|
156
|
+
global___TruncatedSparkRelation = TruncatedSparkRelation
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
import base64
|
|
2
|
+
import logging
|
|
3
|
+
|
|
4
|
+
import pyspark.sql.connect.proto.base_pb2 as spark_proto
|
|
5
|
+
from google.protobuf import message
|
|
6
|
+
from google.protobuf.any_pb2 import Any
|
|
7
|
+
|
|
8
|
+
import snowflake.snowpark
|
|
9
|
+
from snowflake.snowpark.context import get_active_session
|
|
10
|
+
from snowflake.snowpark_decoder._internal.proto.generated import (
|
|
11
|
+
DataframeProcessorMsg_pb2 as dp_proto,
|
|
12
|
+
)
|
|
13
|
+
from snowflake.snowpark_decoder.spark_decoder import SparkDecoder
|
|
14
|
+
from spark.connect.envelope_pb2 import ResponseEnvelope
|
|
15
|
+
|
|
16
|
+
logger = logging.getLogger(__name__)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def str2proto(b64_input: str, proto_output: message.Message) -> None:
|
|
20
|
+
decoded = base64.b64decode(b64_input)
|
|
21
|
+
proto_output.ParseFromString(decoded)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def proto2str(proto_input: message.Message) -> str:
|
|
25
|
+
return str(base64.b64encode(proto_input.SerializeToString()), "utf-8")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class DataframeProcessorSession:
|
|
29
|
+
"""
|
|
30
|
+
The Dataframe Processor Session provides session context for dataframe requests, internally it
|
|
31
|
+
wraps the snowpark session object.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
_instance = None # Class-level variable to store the instance
|
|
35
|
+
|
|
36
|
+
@classmethod
|
|
37
|
+
def get_instance(
|
|
38
|
+
cls,
|
|
39
|
+
dataframe_type: dp_proto.DataframeType,
|
|
40
|
+
session: snowflake.snowpark.Session = None,
|
|
41
|
+
) -> "DataframeProcessorSession":
|
|
42
|
+
"""Returns the singleton instance of the DataframeProcessorSession."""
|
|
43
|
+
if cls._instance is None:
|
|
44
|
+
if session is None:
|
|
45
|
+
session = get_active_session()
|
|
46
|
+
cls._instance = cls(session, dataframe_type)
|
|
47
|
+
return cls._instance
|
|
48
|
+
|
|
49
|
+
def __init__(
|
|
50
|
+
self,
|
|
51
|
+
session: snowflake.snowpark.Session,
|
|
52
|
+
dataframe_type: dp_proto.DataframeType,
|
|
53
|
+
) -> None:
|
|
54
|
+
"""
|
|
55
|
+
Initializes optional Snowpark session to connect to.
|
|
56
|
+
Args:
|
|
57
|
+
session: the Snowpark session to be used.
|
|
58
|
+
dataframe_type: the type of dataframe to be processed.
|
|
59
|
+
"""
|
|
60
|
+
session.ast_enabled = False
|
|
61
|
+
self._session = session
|
|
62
|
+
self._dataframe_type = dataframe_type
|
|
63
|
+
if dataframe_type == dp_proto.SPARK_CONNECT:
|
|
64
|
+
self._decoder = SparkDecoder(self._session)
|
|
65
|
+
else:
|
|
66
|
+
raise RuntimeError(f"Invalid dataframe type: {type}")
|
|
67
|
+
|
|
68
|
+
def request(self, req_base64: str) -> str:
|
|
69
|
+
"""
|
|
70
|
+
The only public method to generate response from a dataframe processor
|
|
71
|
+
request.
|
|
72
|
+
|
|
73
|
+
:param req_base64: the request string encoded by base64
|
|
74
|
+
:return: the response string encoded by base64
|
|
75
|
+
"""
|
|
76
|
+
try:
|
|
77
|
+
dp_req_proto = dp_proto.Request()
|
|
78
|
+
str2proto(req_base64, dp_req_proto)
|
|
79
|
+
rid = dp_req_proto.request_id
|
|
80
|
+
|
|
81
|
+
any_msg = dp_req_proto.payload
|
|
82
|
+
if any_msg.Is(spark_proto.ConfigRequest.DESCRIPTOR):
|
|
83
|
+
request = spark_proto.ConfigRequest()
|
|
84
|
+
elif any_msg.Is(spark_proto.ExecutePlanRequest.DESCRIPTOR):
|
|
85
|
+
request = spark_proto.ExecutePlanRequest()
|
|
86
|
+
elif any_msg.Is(spark_proto.AnalyzePlanRequest.DESCRIPTOR):
|
|
87
|
+
request = spark_proto.AnalyzePlanRequest()
|
|
88
|
+
else:
|
|
89
|
+
raise NotImplementedError("Unknown request type")
|
|
90
|
+
dp_req_proto.payload.Unpack(request)
|
|
91
|
+
result = self._decoder.request(request)
|
|
92
|
+
|
|
93
|
+
assert isinstance(result, ResponseEnvelope)
|
|
94
|
+
code = (
|
|
95
|
+
dp_proto.Response.StatusCode.EXECUTION_ERROR
|
|
96
|
+
if result.WhichOneof("response_type") == "status"
|
|
97
|
+
else dp_proto.Response.StatusCode.OK
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
payload = Any()
|
|
101
|
+
payload.Pack(result)
|
|
102
|
+
dp_res_proto = dp_proto.Response(
|
|
103
|
+
code=code,
|
|
104
|
+
payload=payload,
|
|
105
|
+
dataframe_type=dp_req_proto.dataframe_type,
|
|
106
|
+
request_id=rid,
|
|
107
|
+
)
|
|
108
|
+
return proto2str(dp_res_proto)
|
|
109
|
+
except Exception:
|
|
110
|
+
# TODO: SNOW-1857056 error handling
|
|
111
|
+
raise
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from typing import Union
|
|
3
|
+
|
|
4
|
+
import pyspark.sql.connect.proto.base_pb2 as proto
|
|
5
|
+
|
|
6
|
+
import snowflake.snowpark_connect.tcm as tcm
|
|
7
|
+
from snowflake.snowpark import Session
|
|
8
|
+
from snowflake.snowpark_connect.error.error_utils import build_grpc_error_response
|
|
9
|
+
from snowflake.snowpark_connect.execute_plan.map_execution_root import QueryResult
|
|
10
|
+
from snowflake.snowpark_connect.server import SnowflakeConnectServicer, start_session
|
|
11
|
+
from spark.connect.envelope_pb2 import DataframeQueryResult, ResponseEnvelope
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class SparkDecoder:
|
|
15
|
+
"""
|
|
16
|
+
Spark Decoder is the main snowflake server entry point that accepts spark connect requests.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
REQUEST_TYPES = [
|
|
20
|
+
proto.ExecutePlanRequest,
|
|
21
|
+
proto.ConfigRequest,
|
|
22
|
+
proto.AnalyzePlanRequest,
|
|
23
|
+
]
|
|
24
|
+
|
|
25
|
+
def __init__(self, session: Session) -> None:
|
|
26
|
+
self.session = session
|
|
27
|
+
# set SPARK_LOCAL_HOSTNAME to avoid network lookup in sandbox
|
|
28
|
+
os.environ["SPARK_LOCAL_HOSTNAME"] = "127.0.0.1"
|
|
29
|
+
tcm.TCM_MODE = True
|
|
30
|
+
start_session(is_daemon=False, snowpark_session=self.session)
|
|
31
|
+
self.servicer = SnowflakeConnectServicer()
|
|
32
|
+
|
|
33
|
+
def request(
|
|
34
|
+
self,
|
|
35
|
+
request: Union[
|
|
36
|
+
proto.ExecutePlanRequest, proto.AnalyzePlanRequest, proto.ConfigRequest
|
|
37
|
+
],
|
|
38
|
+
) -> ResponseEnvelope:
|
|
39
|
+
try:
|
|
40
|
+
ctx = Context()
|
|
41
|
+
match request:
|
|
42
|
+
case proto.ExecutePlanRequest():
|
|
43
|
+
res = self.servicer.ExecutePlan(request, ctx)
|
|
44
|
+
result = next(res)
|
|
45
|
+
if isinstance(result, QueryResult):
|
|
46
|
+
return ResponseEnvelope(
|
|
47
|
+
dataframe_query_result=DataframeQueryResult(
|
|
48
|
+
result_job_uuid=result.query_id,
|
|
49
|
+
arrow_schema=result.arrow_schema,
|
|
50
|
+
spark_schema=result.spark_schema,
|
|
51
|
+
)
|
|
52
|
+
)
|
|
53
|
+
else:
|
|
54
|
+
return ResponseEnvelope(execute_plan_response=result)
|
|
55
|
+
case proto.AnalyzePlanRequest():
|
|
56
|
+
return ResponseEnvelope(
|
|
57
|
+
analyze_plan_response=self.servicer.AnalyzePlan(request, ctx)
|
|
58
|
+
)
|
|
59
|
+
case proto.ConfigRequest():
|
|
60
|
+
return ResponseEnvelope(
|
|
61
|
+
config_response=self.servicer.Config(request, ctx)
|
|
62
|
+
)
|
|
63
|
+
case _:
|
|
64
|
+
raise NotImplementedError(
|
|
65
|
+
"Unknown request type: %s" % type(request)
|
|
66
|
+
)
|
|
67
|
+
except Exception as e:
|
|
68
|
+
return ResponseEnvelope(status=build_grpc_error_response(e))
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class Context:
|
|
72
|
+
def __init__(self) -> None:
|
|
73
|
+
pass
|
|
74
|
+
|
|
75
|
+
def abort_with_status(status):
|
|
76
|
+
raise NotImplementedError("abort_with_status is not implemented")
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: snowpark-connect
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.22.1
|
|
4
4
|
Summary: Snowpark Connect for Spark
|
|
5
5
|
Author: Snowflake, Inc
|
|
6
6
|
License: Apache License, Version 2.0
|
|
@@ -16,7 +16,7 @@ Requires-Dist: jpype1
|
|
|
16
16
|
Requires-Dist: protobuf<5.0,>=4.25.3
|
|
17
17
|
Requires-Dist: s3fs>=2025.3.0
|
|
18
18
|
Requires-Dist: snowflake.core<2,>=1.0.5
|
|
19
|
-
Requires-Dist: snowflake-snowpark-python[pandas]!=1.35.0,>=1.34.0
|
|
19
|
+
Requires-Dist: snowflake-snowpark-python[pandas]!=1.35.0,!=1.36.0,>=1.34.0
|
|
20
20
|
Requires-Dist: sqlglot>=26.3.8
|
|
21
21
|
Requires-Dist: jaydebeapi
|
|
22
22
|
Requires-Dist: aiobotocore~=2.23.0
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
snowflake/snowpark_connect/__init__.py,sha256=Sml4x1LTNnxZyw6nnDeJrZWUi3eUAR46Rsw6N-wHUSA,605
|
|
2
2
|
snowflake/snowpark_connect/column_name_handler.py,sha256=69N4D1cwld9afBGEkLHxzoeawDhwYsyTQA8bmrIX8XM,26045
|
|
3
|
-
snowflake/snowpark_connect/config.py,sha256=
|
|
3
|
+
snowflake/snowpark_connect/config.py,sha256=XzPAdYXzXkFT7pe76b0LNTnoNoMahqi3s03Cr5Kzmtc,24447
|
|
4
4
|
snowflake/snowpark_connect/constants.py,sha256=33qit4s6E2_OPIUNSPUOelfHHkQIPYtW1zjxe9hoY-M,1409
|
|
5
5
|
snowflake/snowpark_connect/control_server.py,sha256=mz3huYo84hgqUB6maZxu3LYyGq7vVL1nv7-7-MjuSYY,1956
|
|
6
6
|
snowflake/snowpark_connect/dataframe_container.py,sha256=reVGpNbKwmQc1CTjksypmxwKhBpenan_r1XxkMSXXFo,8152
|
|
@@ -8,17 +8,17 @@ snowflake/snowpark_connect/dataframe_name_handler.py,sha256=aR-CpdGsN2d6tNW0H_F9
|
|
|
8
8
|
snowflake/snowpark_connect/date_time_format_mapping.py,sha256=qtQ-JTGR1VRWM2oxM1aYggE_g-BNouffeHxmCk89xkk,16809
|
|
9
9
|
snowflake/snowpark_connect/empty_dataframe.py,sha256=aKO6JkYnArWCpLGcn9BzvTspw2k_c6eAM0mQImAY0J0,428
|
|
10
10
|
snowflake/snowpark_connect/resources_initializer.py,sha256=fzo0Xjlvn-GToot2g9dSuNwUOcjUU3GD-55h1xqlRy8,2969
|
|
11
|
-
snowflake/snowpark_connect/server.py,sha256=
|
|
11
|
+
snowflake/snowpark_connect/server.py,sha256=ZknoIYR47pXSQgUXgMkv9pczRlCDtw6AeNzg4751CLI,49575
|
|
12
12
|
snowflake/snowpark_connect/start_server.py,sha256=udegO0rk2FeSnXsIcCIYQW3VRlGDjB_cU8lJ8xSzuM8,942
|
|
13
13
|
snowflake/snowpark_connect/tcm.py,sha256=ftncZFbVO-uyWMhF1_HYKQykB7KobHEYoyQsYbQj1EM,203
|
|
14
|
-
snowflake/snowpark_connect/type_mapping.py,sha256=
|
|
14
|
+
snowflake/snowpark_connect/type_mapping.py,sha256=MgU2_3BQ0qE-32zs73wePel14wZuqyJkZwlv6fvVn3w,41474
|
|
15
15
|
snowflake/snowpark_connect/typed_column.py,sha256=4rdODZqOGW60zVhmuhtljh9ijlHappRXhE5Mgr35CVs,3288
|
|
16
|
-
snowflake/snowpark_connect/version.py,sha256=
|
|
16
|
+
snowflake/snowpark_connect/version.py,sha256=ulrmIkMPOytVZQuZ6gQEMU4i6zSK3KqbxYKDhUZ4r90,121
|
|
17
17
|
snowflake/snowpark_connect/analyze_plan/__init__.py,sha256=xsIE96jDASko3F-MeNf4T4Gg5ufthS8CejeiJDfri0M,76
|
|
18
18
|
snowflake/snowpark_connect/analyze_plan/map_tree_string.py,sha256=Q3ZD-Z7uForrF7W3mSAjwaiEcIv2KDXr5jPfVbromVg,1470
|
|
19
19
|
snowflake/snowpark_connect/error/__init__.py,sha256=oQo6k4zztLmNF1c5IvJLcS99J6RWY9KBTN3RJ2pKimg,249
|
|
20
20
|
snowflake/snowpark_connect/error/error_mapping.py,sha256=vdnLOU1Sqpocpu_uCXjfhivutgD3yf60U4D31DJ31ng,195361
|
|
21
|
-
snowflake/snowpark_connect/error/error_utils.py,sha256=
|
|
21
|
+
snowflake/snowpark_connect/error/error_utils.py,sha256=p76vZlXOz9CTl34Xq3BfikPh9vWD-u46mi0bAt2quKw,14596
|
|
22
22
|
snowflake/snowpark_connect/error/exceptions.py,sha256=7Kazwf_tJ5gBHEoIhRjw3d8UC_icHuZMIb3EFd6s-Io,617
|
|
23
23
|
snowflake/snowpark_connect/execute_plan/__init__.py,sha256=xsIE96jDASko3F-MeNf4T4Gg5ufthS8CejeiJDfri0M,76
|
|
24
24
|
snowflake/snowpark_connect/execute_plan/map_execution_command.py,sha256=V25tke22PT92nCJVjDTjeaw0UxspYaOus-9BYb4gfX8,8150
|
|
@@ -30,10 +30,10 @@ snowflake/snowpark_connect/expression/map_cast.py,sha256=cqQ4jaZrjIl_Pviug76zMbk
|
|
|
30
30
|
snowflake/snowpark_connect/expression/map_expression.py,sha256=bURxgMidLggf40Vg_GaQceJThhSgYiCfrGMsvcum8Qg,13435
|
|
31
31
|
snowflake/snowpark_connect/expression/map_extension.py,sha256=W_rTazMY6lpOpyVjzF96xev0iQGd8zudn02DZb3moCw,4613
|
|
32
32
|
snowflake/snowpark_connect/expression/map_sql_expression.py,sha256=3ej7ff7fSpPaKzXbUo_7oNVAPhcx2j4JeVYywbjAJXc,24963
|
|
33
|
-
snowflake/snowpark_connect/expression/map_udf.py,sha256=
|
|
33
|
+
snowflake/snowpark_connect/expression/map_udf.py,sha256=GYDyO4_3VJXRtO2fIP5d1kHYuM2FGSY4oN-gHfx4xCc,5639
|
|
34
34
|
snowflake/snowpark_connect/expression/map_unresolved_attribute.py,sha256=OcgBo8jC0uUfuv5GQiUOuJ4um4fkOUdZsaLpR6uwL40,10178
|
|
35
35
|
snowflake/snowpark_connect/expression/map_unresolved_extract_value.py,sha256=QjPl1-FD8idSfNF1xmQREG5sI1OGg3Bwix3zvmLNq1E,3508
|
|
36
|
-
snowflake/snowpark_connect/expression/map_unresolved_function.py,sha256=
|
|
36
|
+
snowflake/snowpark_connect/expression/map_unresolved_function.py,sha256=COQ3fHX-DsvoJeGgQg3oRNWhzDmsL9ax1T6IL3hdigA,449487
|
|
37
37
|
snowflake/snowpark_connect/expression/map_unresolved_star.py,sha256=gecSxJ9flsn9chTXmYHP8nLzs_209xnVlyNwANDXRFg,8820
|
|
38
38
|
snowflake/snowpark_connect/expression/map_update_fields.py,sha256=bMuZjCOE5wgrZv3ApsoiwEUv-lPgnMWUk-DFX11QuX0,6710
|
|
39
39
|
snowflake/snowpark_connect/expression/map_window_function.py,sha256=apJxtWSnLogMG0F_En67JoGrGRGYQ0I8O0xPRk4TR_o,12152
|
|
@@ -62,7 +62,6 @@ snowflake/snowpark_connect/includes/jars/jackson-core-asl-1.9.13.jar,sha256=RAqc
|
|
|
62
62
|
snowflake/snowpark_connect/includes/jars/jackson-databind-2.15.2.jar,sha256=DrL9rW5Aq4gyp4ybIvWBlt2XBZTo09Wibq2HhHxPOpY,1620088
|
|
63
63
|
snowflake/snowpark_connect/includes/jars/jackson-dataformat-yaml-2.15.2.jar,sha256=N3lcwejLlLGNhg3Dq9Llk2F85AIUmuRaqJ7Yv7iByFE,54630
|
|
64
64
|
snowflake/snowpark_connect/includes/jars/jackson-datatype-jsr310-2.15.2.jar,sha256=dXTIGtVwR272qtJvQZKI_UZnM_MxW-4wEvLynJ3ACMg,122937
|
|
65
|
-
snowflake/snowpark_connect/includes/jars/jackson-mapper-asl-1.9.13.jar,sha256=dOegenby7breKTEqWi68z6AZEovAIezjhW12GX6b4MI,780664
|
|
66
65
|
snowflake/snowpark_connect/includes/jars/jackson-module-scala_2.12-2.15.2.jar,sha256=gSWexnMPbUuF1VhsnqpudaaIWzS_ylcr4DEmqReXGf4,513968
|
|
67
66
|
snowflake/snowpark_connect/includes/jars/json4s-ast_2.12-3.7.0-M11.jar,sha256=-TiMCtSpab8aGGfoXOJO2HyloQlTQrlrNcNW_ZcbVE0,89855
|
|
68
67
|
snowflake/snowpark_connect/includes/jars/json4s-core_2.12-3.7.0-M11.jar,sha256=L9jmajcVVlKUxO210lFOf_WaFJYMf43YTPCZFmGkihA,526286
|
|
@@ -806,27 +805,29 @@ snowflake/snowpark_connect/proto/control_pb2.pyi,sha256=DQVcgVtvo1W1tpdi8_ogBO6D
|
|
|
806
805
|
snowflake/snowpark_connect/proto/control_pb2_grpc.py,sha256=IPHf2f27M1HElF55HH-Kvd8WCFRo6SDj_EjEzFEe_rg,5444
|
|
807
806
|
snowflake/snowpark_connect/proto/snowflake_expression_ext_pb2.py,sha256=NSOT7lvgSbfN8iwxl1nmbJYSjoR3eGYEtYzlnrXtL88,3127
|
|
808
807
|
snowflake/snowpark_connect/proto/snowflake_expression_ext_pb2.pyi,sha256=G6wa6Ei-LHgZH4mXG0vOAli5D5JIJQO8RQo8E_4ui4o,3831
|
|
808
|
+
snowflake/snowpark_connect/proto/snowflake_expression_ext_pb2_grpc.py,sha256=1oboBPFxaTEXt9Aw7EAj8gXHDCNMhZD2VXqocC9l_gk,159
|
|
809
809
|
snowflake/snowpark_connect/proto/snowflake_rdd_pb2.pyi,sha256=IA_TC6cYM50qTlpSL0UwhmbNkNYgEYuNcZLYTxbGfBc,1780
|
|
810
810
|
snowflake/snowpark_connect/proto/snowflake_relation_ext_pb2.py,sha256=frZtEPyXWgq3g8-iq-dM0ZB1qWE04y1QswqyFmQCQtk,4805
|
|
811
811
|
snowflake/snowpark_connect/proto/snowflake_relation_ext_pb2.pyi,sha256=eskQnX2EcIEqjtcjX6XH9sIOgunDYJhO44V8ggEnzns,7100
|
|
812
|
+
snowflake/snowpark_connect/proto/snowflake_relation_ext_pb2_grpc.py,sha256=1oboBPFxaTEXt9Aw7EAj8gXHDCNMhZD2VXqocC9l_gk,159
|
|
812
813
|
snowflake/snowpark_connect/relation/__init__.py,sha256=xsIE96jDASko3F-MeNf4T4Gg5ufthS8CejeiJDfri0M,76
|
|
813
814
|
snowflake/snowpark_connect/relation/io_utils.py,sha256=bFW6lxjGnXiOW_OOqS7kPsuwjybVNpl8-Z9Xa6sQp20,1821
|
|
814
|
-
snowflake/snowpark_connect/relation/map_aggregate.py,sha256=
|
|
815
|
+
snowflake/snowpark_connect/relation/map_aggregate.py,sha256=_DePa2LtH3noTfRMdz6gNvVNZmEzrLs1qhc7lGdC_Ts,16620
|
|
815
816
|
snowflake/snowpark_connect/relation/map_catalog.py,sha256=mcx6An4fqHAxy2OhOC66Xe_0aRtYPDGkBEgMK5CfaXU,5822
|
|
816
|
-
snowflake/snowpark_connect/relation/map_column_ops.py,sha256=
|
|
817
|
+
snowflake/snowpark_connect/relation/map_column_ops.py,sha256=uTctkn-j0l93AW0PasqGEZSaZj9P4kRsflEiC6cqRcg,48339
|
|
817
818
|
snowflake/snowpark_connect/relation/map_crosstab.py,sha256=H_J8-IARK6zMEUFrOjKif1St6M20gvBAnP0EuArFHGg,2422
|
|
818
819
|
snowflake/snowpark_connect/relation/map_extension.py,sha256=AcNqJDQGwMeXnAVCSLfMm9sy41y9BvWbXCiYhDvaDks,18422
|
|
819
820
|
snowflake/snowpark_connect/relation/map_join.py,sha256=YGOy6nmpb0mqw8D3TzsV4bDlvaq6QFYPwfF5S6YgydU,15201
|
|
820
821
|
snowflake/snowpark_connect/relation/map_local_relation.py,sha256=_UVWt6LhZWyNh0k0FBYCCsheb9PT-AkWkbT77-fxZBI,14871
|
|
821
|
-
snowflake/snowpark_connect/relation/map_map_partitions.py,sha256=
|
|
822
|
+
snowflake/snowpark_connect/relation/map_map_partitions.py,sha256=AMTz9S2PWjsCDM-Lz_MUTRQ-TUSesdNb-wFa7BUvmB8,5690
|
|
822
823
|
snowflake/snowpark_connect/relation/map_relation.py,sha256=iH-VzP6-dwFlTOOYABiMM7xsgrYuQ8BV3UtvjS39w98,11646
|
|
823
824
|
snowflake/snowpark_connect/relation/map_row_ops.py,sha256=m9ulYPfqPtlBtxXCUXMh5EMGPIGiOpat_JsfHszMKI0,30061
|
|
824
825
|
snowflake/snowpark_connect/relation/map_sample_by.py,sha256=8ALQbeUsB89sI3uiUFqG3w1A4TtOzOAL4umdKp6-c38,1530
|
|
825
826
|
snowflake/snowpark_connect/relation/map_show_string.py,sha256=fuYCuThp7V3VPb5NETJvW0JDf9Xv2qCxdvBmlbHgF7c,3254
|
|
826
|
-
snowflake/snowpark_connect/relation/map_sql.py,sha256=
|
|
827
|
+
snowflake/snowpark_connect/relation/map_sql.py,sha256=GCldOd0sGui4mQnD_snK8_gSTBkNuafuG6rTRIa1lg8,78248
|
|
827
828
|
snowflake/snowpark_connect/relation/map_stats.py,sha256=kqRYvix8RfluTKx1cAy9JhBUv6arYQHgfxpP1R4QwBM,13985
|
|
828
829
|
snowflake/snowpark_connect/relation/map_subquery_alias.py,sha256=rHgE9XUzuWWkjNPtJz3Sxzz2aFo690paHKZh9frqPXk,1456
|
|
829
|
-
snowflake/snowpark_connect/relation/map_udtf.py,sha256=
|
|
830
|
+
snowflake/snowpark_connect/relation/map_udtf.py,sha256=MTRgYFo7qGMCthVqLFnugTMJ0E2nsrp0sMgtfff1U2E,11278
|
|
830
831
|
snowflake/snowpark_connect/relation/stage_locator.py,sha256=c30Z4N_xFavaL5XhIl9sCbhrgn6BLhLbh2xXEaj-QmM,6885
|
|
831
832
|
snowflake/snowpark_connect/relation/utils.py,sha256=7Gjhyw0djcF9a5kmi1pQHTdUpW5pKI94VaH7L3OvpU4,6956
|
|
832
833
|
snowflake/snowpark_connect/relation/catalogs/__init__.py,sha256=0yJ5Nfg7SIxudI0P7_U5EWPyiTpkMet8tSq-IwutSZo,265
|
|
@@ -838,16 +839,16 @@ snowflake/snowpark_connect/relation/read/jdbc_read_dbapi.py,sha256=nwvBYwdDTRQc8
|
|
|
838
839
|
snowflake/snowpark_connect/relation/read/map_read.py,sha256=mIGHYcK0S1N2CeJ7xNIpAL0LjxByYmrqeSObyqNNze4,14884
|
|
839
840
|
snowflake/snowpark_connect/relation/read/map_read_csv.py,sha256=aAhzWprNeSTnb_4FyBjQDmDUFu9H5p0CQP6sBQHUCQI,4934
|
|
840
841
|
snowflake/snowpark_connect/relation/read/map_read_jdbc.py,sha256=9sUrrEGMLWUZ9hFIMD6S0N5Uoq-DkNFNuObah8bVlqU,3958
|
|
841
|
-
snowflake/snowpark_connect/relation/read/map_read_json.py,sha256=
|
|
842
|
+
snowflake/snowpark_connect/relation/read/map_read_json.py,sha256=TZbpnAY3vO-gWDcx8xVoT0kDDTTgoGixhn0EMt6l2OQ,11974
|
|
842
843
|
snowflake/snowpark_connect/relation/read/map_read_parquet.py,sha256=jhn0edxGbdSLCfFiTW1VIrkezLiIo-ii3lDC7rtoY2E,6919
|
|
843
844
|
snowflake/snowpark_connect/relation/read/map_read_socket.py,sha256=yg7aO7jXMOyLQ9k3-abywNN2VAgvtvix2V8LsR_XQO4,2267
|
|
844
845
|
snowflake/snowpark_connect/relation/read/map_read_table.py,sha256=6R2ywUQCM24dWubKco4clCn2HPbi5rPb74FsRIRmDzs,4611
|
|
845
846
|
snowflake/snowpark_connect/relation/read/map_read_text.py,sha256=Vcm6EY3gyRPucdAXcXcu2mREcgq4N_4h3NDKNPhqsHY,3434
|
|
846
|
-
snowflake/snowpark_connect/relation/read/reader_config.py,sha256
|
|
847
|
+
snowflake/snowpark_connect/relation/read/reader_config.py,sha256=-TZuJn20jnTz_d6thxsT8KotPVHe5fdVtO99hep3098,15933
|
|
847
848
|
snowflake/snowpark_connect/relation/read/utils.py,sha256=rIIM6d2WXHh7MLGyHNiRc9tS8b0dmyFQr7rHepIYJOU,4111
|
|
848
849
|
snowflake/snowpark_connect/relation/write/__init__.py,sha256=xsIE96jDASko3F-MeNf4T4Gg5ufthS8CejeiJDfri0M,76
|
|
849
850
|
snowflake/snowpark_connect/relation/write/jdbc_write_dbapi.py,sha256=GI9FyGZuQQNV-6Q8Ob-Xr0im3iAPdH-Jkyx8bjwbOuE,11931
|
|
850
|
-
snowflake/snowpark_connect/relation/write/map_write.py,sha256=
|
|
851
|
+
snowflake/snowpark_connect/relation/write/map_write.py,sha256=lliaNaleRQ2ve994kO3w3QL-vORUG0d9JtH-tPAET4U,25027
|
|
851
852
|
snowflake/snowpark_connect/relation/write/map_write_jdbc.py,sha256=1nOWRgjtZzfRwnSRGFP9V6mqBVlGhSBr2KHGHbe4JMU,1404
|
|
852
853
|
snowflake/snowpark_connect/resources/java_udfs-1.0-SNAPSHOT.jar,sha256=tVyOp6tXxu9nm6SDufwQiGzfH3pnuh_7PowsMZxOolY,9773
|
|
853
854
|
snowflake/snowpark_connect/utils/__init__.py,sha256=xsIE96jDASko3F-MeNf4T4Gg5ufthS8CejeiJDfri0M,76
|
|
@@ -855,28 +856,38 @@ snowflake/snowpark_connect/utils/artifacts.py,sha256=KY-tF1Kyksg65CpivjvEDmOko9R
|
|
|
855
856
|
snowflake/snowpark_connect/utils/cache.py,sha256=bAyoNBW6Z1ui9BuppDywbQeG6fdju4L-owFHzySOTnk,3382
|
|
856
857
|
snowflake/snowpark_connect/utils/concurrent.py,sha256=cuPItxiDgqIeZ35881gd3VI1rwQTTu2ldsgw3ZcGqDk,3448
|
|
857
858
|
snowflake/snowpark_connect/utils/context.py,sha256=UFo9TeQmBuJKPy_rk_F48JwPpiArw2A5TuWD0g3Fp2c,11871
|
|
858
|
-
snowflake/snowpark_connect/utils/describe_query_cache.py,sha256=
|
|
859
|
+
snowflake/snowpark_connect/utils/describe_query_cache.py,sha256=2VcPgGP9bUpdIhnN2s_MOG8oGHKX0hS0rT7Y26MJb3A,9001
|
|
859
860
|
snowflake/snowpark_connect/utils/identifiers.py,sha256=YgtVIQGuUnnTiNdtRficdBwUICWaWkJltjOPnTnfrak,3881
|
|
860
861
|
snowflake/snowpark_connect/utils/interrupt.py,sha256=_awhdrzF1KQO-EQThneEcfMg3Zxed4p3HtMpkcAb6ek,2790
|
|
861
862
|
snowflake/snowpark_connect/utils/io_utils.py,sha256=noBlKpJvzEA6iwLjFgBVGlCLlzjZ16-w0fsGimTyBAQ,1039
|
|
862
863
|
snowflake/snowpark_connect/utils/pandas_udtf_utils.py,sha256=QwdLGLg5bX0JJTrWKfL4Ou4MIp443cryEbrZNBrulNE,4207
|
|
863
864
|
snowflake/snowpark_connect/utils/profiling.py,sha256=ttdHzQUYarvTqJASLNuKFIax7ejO39Tv1mHKl0QjRkg,1519
|
|
864
|
-
snowflake/snowpark_connect/utils/session.py,sha256=
|
|
865
|
+
snowflake/snowpark_connect/utils/session.py,sha256=4VzWDDllNB_5G0D1tYNnZ-5gUc-_GcFLUqsRNdgJCzM,6705
|
|
865
866
|
snowflake/snowpark_connect/utils/snowpark_connect_logging.py,sha256=23bvbALGqixJ3Ap9QWM3OpcKNK-sog2mr9liSmvwqYU,1123
|
|
866
|
-
snowflake/snowpark_connect/utils/telemetry.py,sha256=
|
|
867
|
+
snowflake/snowpark_connect/utils/telemetry.py,sha256=k41tvbOykIEpNfBYTrmqg0cRwiAQ3b7obhsEzU417aQ,21100
|
|
867
868
|
snowflake/snowpark_connect/utils/udf_cache.py,sha256=8K7kASEhvpnp-l1hjzovjyboUzKctDq7PiGXRcNv6Lg,12125
|
|
868
869
|
snowflake/snowpark_connect/utils/udf_helper.py,sha256=AC5SkUFy9I7-HYf8awuPX33eyWvGen84Vl7fAU0RXng,12513
|
|
869
870
|
snowflake/snowpark_connect/utils/udf_utils.py,sha256=zWFc1G-3GjfDDKfzUX8sGBzEf2IMAMRs4NLSZ81Gz-8,13249
|
|
870
871
|
snowflake/snowpark_connect/utils/udtf_helper.py,sha256=nTWbrFMc4hHLkalT4nJIsC_c00T6GFkB55JHOw1D-wg,14571
|
|
871
872
|
snowflake/snowpark_connect/utils/udtf_utils.py,sha256=SZp6soES4TF-5FnfJPJTT6igAelo67KXUjWFtMpxFlI,33109
|
|
873
|
+
snowflake/snowpark_connect/utils/udxf_import_utils.py,sha256=pPtcaGsyh0tUdy0aAvNqTj04jqPKlEcGmvaZDP9O8Gc,536
|
|
872
874
|
snowflake/snowpark_connect/utils/xxhash64.py,sha256=ysJRxhBPf25LeNhM1RK_H36MWl6q6C6vBRHa-jIna_A,7477
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
snowpark_connect-0.
|
|
879
|
-
snowpark_connect-0.
|
|
880
|
-
snowpark_connect-0.
|
|
881
|
-
snowpark_connect-0.
|
|
882
|
-
snowpark_connect-0.
|
|
875
|
+
snowflake/snowpark_decoder/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
876
|
+
snowflake/snowpark_decoder/dp_session.py,sha256=HIr3TfKgYl5zqaGR5xpFU9ZVkcaTB9I8xCAzbv_-gY0,3923
|
|
877
|
+
snowflake/snowpark_decoder/spark_decoder.py,sha256=trMwTpnJqNFjCkjWVTFWmZ3mUNh9N3Dj20JbuqsLUFQ,2879
|
|
878
|
+
snowflake/snowpark_decoder/_internal/proto/generated/DataframeProcessorMsg_pb2.py,sha256=2eSDqeyfMvmIJ6_rF663DrEe1dg_anrP4OpVJNTJHaQ,2598
|
|
879
|
+
snowflake/snowpark_decoder/_internal/proto/generated/DataframeProcessorMsg_pb2.pyi,sha256=aIH23k52bXdw5vO3RtM5UcOjDPaWsJFx1SRUSk3qOK8,6142
|
|
880
|
+
snowpark_connect-0.22.1.data/scripts/snowpark-connect,sha256=yZ94KqbWACxnwV8mpg8NjILvvRNjnF8B3cs3ZFNuIM4,1546
|
|
881
|
+
snowpark_connect-0.22.1.data/scripts/snowpark-session,sha256=NMAHSonTo-nmOZSkQNlszUC0jLJ8QWEDUsUmMe2UAOw,190
|
|
882
|
+
snowpark_connect-0.22.1.data/scripts/snowpark-submit,sha256=Zd98H9W_d0dIqMSkQLdHyW5G3myxF0t4c3vNBt2nD6A,12056
|
|
883
|
+
snowpark_connect-0.22.1.dist-info/licenses/LICENSE-binary,sha256=fmBlX39HwTlBUyiKEznaLZGuxQy-7ndLLG_rTXjF02Y,22916
|
|
884
|
+
snowpark_connect-0.22.1.dist-info/licenses/LICENSE.txt,sha256=Ff9cPv4xu0z7bnMTHzo4vDncOShsy33w4oJMA2xjn6c,11365
|
|
885
|
+
snowpark_connect-0.22.1.dist-info/licenses/NOTICE-binary,sha256=elMF8brgGNJwOz8YdorzBF6-U8ZhR8F-77FfGkZng7U,57843
|
|
886
|
+
spark/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
887
|
+
spark/connect/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
888
|
+
spark/connect/envelope_pb2.py,sha256=7Gc6OUA3vaCuTCIKamb_Iiw7W9jPTcWNEv1im20eWHM,2726
|
|
889
|
+
spark/connect/envelope_pb2.pyi,sha256=VXTJSPpcxzB_dWqVdvPY4KkPhJfh0WmkX7SNHWoLhx0,3358
|
|
890
|
+
snowpark_connect-0.22.1.dist-info/METADATA,sha256=5Zua2Lfyv6-CFAYATEeNOrgRbKebBoZGxGCV1ThE-RM,1604
|
|
891
|
+
snowpark_connect-0.22.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
892
|
+
snowpark_connect-0.22.1.dist-info/top_level.txt,sha256=ExnWqVpoTHRG99fu_AxXZVOz8c-De7nNu0yFCGylM8I,16
|
|
893
|
+
snowpark_connect-0.22.1.dist-info/RECORD,,
|
spark/__init__.py
ADDED
|
File without changes
|
|
File without changes
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
|
3
|
+
# source: envelope.proto
|
|
4
|
+
# Protobuf Python Version: 4.25.1
|
|
5
|
+
"""Generated protocol buffer code."""
|
|
6
|
+
from google.protobuf import descriptor as _descriptor
|
|
7
|
+
from google.protobuf import descriptor_pool as _descriptor_pool
|
|
8
|
+
from google.protobuf import symbol_database as _symbol_database
|
|
9
|
+
from google.protobuf.internal import builder as _builder
|
|
10
|
+
# @@protoc_insertion_point(imports)
|
|
11
|
+
|
|
12
|
+
_sym_db = _symbol_database.Default()
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
from pyspark.sql.connect.proto import base_pb2 as spark_dot_connect_dot_base__pb2
|
|
16
|
+
from google.rpc import status_pb2 as google_dot_rpc_dot_status__pb2
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0e\x65nvelope.proto\x12\rspark.connect\x1a\x18spark/connect/base.proto\x1a\x17google/rpc/status.proto\"[\n\x14\x44\x61taframeQueryResult\x12\x17\n\x0fresult_job_uuid\x18\x01 \x01(\t\x12\x14\n\x0c\x61rrow_schema\x18\x02 \x01(\x0c\x12\x14\n\x0cspark_schema\x18\x03 \x01(\x0c\"\xb4\x05\n\x10ResponseEnvelope\x12\x43\n\x15\x65xecute_plan_response\x18\x01 \x01(\x0b\x32\".spark.connect.ExecutePlanResponseH\x00\x12\x43\n\x15\x61nalyze_plan_response\x18\x02 \x01(\x0b\x32\".spark.connect.AnalyzePlanResponseH\x00\x12\x38\n\x0f\x63onfig_response\x18\x03 \x01(\x0b\x32\x1d.spark.connect.ConfigResponseH\x00\x12\x45\n\x16\x61\x64\x64_artifacts_response\x18\x04 \x01(\x0b\x32#.spark.connect.AddArtifactsResponseH\x00\x12K\n\x18\x61rtifact_status_response\x18\x05 \x01(\x0b\x32\'.spark.connect.ArtifactStatusesResponseH\x00\x12>\n\x12interrupt_response\x18\x06 \x01(\x0b\x32 .spark.connect.InterruptResponseH\x00\x12I\n\x18release_execute_response\x18\x07 \x01(\x0b\x32%.spark.connect.ReleaseExecuteResponseH\x00\x12$\n\x06status\x18\x0b \x01(\x0b\x32\x12.google.rpc.StatusH\x00\x12\x45\n\x16\x64\x61taframe_query_result\x18\x0c \x01(\x0b\x32#.spark.connect.DataframeQueryResultH\x00\x12\x10\n\x08query_id\x18\x08 \x01(\t\x12\x12\n\nrequest_id\x18\t \x01(\t\x12\x19\n\x11result_partitions\x18\n \x03(\tB\x0f\n\rresponse_typeB9\n!com.snowflake.spark.connect.protoP\x01Z\x12internal/generatedb\x06proto3')
|
|
20
|
+
|
|
21
|
+
_globals = globals()
|
|
22
|
+
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
|
|
23
|
+
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'envelope_pb2', _globals)
|
|
24
|
+
if _descriptor._USE_C_DESCRIPTORS == False:
|
|
25
|
+
_globals['DESCRIPTOR']._options = None
|
|
26
|
+
_globals['DESCRIPTOR']._serialized_options = b'\n!com.snowflake.spark.connect.protoP\001Z\022internal/generated'
|
|
27
|
+
_globals['_DATAFRAMEQUERYRESULT']._serialized_start=84
|
|
28
|
+
_globals['_DATAFRAMEQUERYRESULT']._serialized_end=175
|
|
29
|
+
_globals['_RESPONSEENVELOPE']._serialized_start=178
|
|
30
|
+
_globals['_RESPONSEENVELOPE']._serialized_end=870
|
|
31
|
+
# @@protoc_insertion_point(module_scope)
|