salesforce-data-customcode 6.1.0.dev2__py3-none-any.whl → 6.1.0.dev4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- datacustomcode/__init__.py +5 -0
- datacustomcode/cli.py +29 -4
- datacustomcode/client.py +212 -94
- datacustomcode/config.py +5 -0
- datacustomcode/constants.py +8 -0
- datacustomcode/deploy.py +55 -16
- datacustomcode/io/reader/base.py +42 -0
- datacustomcode/io/writer/base.py +33 -0
- datacustomcode/named_credential/__init__.py +2 -0
- datacustomcode/named_credential/direct/transport.py +6 -6
- datacustomcode/named_credential/errors.py +36 -0
- datacustomcode/run.py +11 -0
- datacustomcode/scan.py +164 -29
- datacustomcode/template.py +13 -1
- datacustomcode/templates/script/examples/streaming_deltas/entrypoint.py +49 -0
- {salesforce_data_customcode-6.1.0.dev2.dist-info → salesforce_data_customcode-6.1.0.dev4.dist-info}/METADATA +40 -3
- {salesforce_data_customcode-6.1.0.dev2.dist-info → salesforce_data_customcode-6.1.0.dev4.dist-info}/RECORD +20 -18
- {salesforce_data_customcode-6.1.0.dev2.dist-info → salesforce_data_customcode-6.1.0.dev4.dist-info}/WHEEL +0 -0
- {salesforce_data_customcode-6.1.0.dev2.dist-info → salesforce_data_customcode-6.1.0.dev4.dist-info}/entry_points.txt +0 -0
- {salesforce_data_customcode-6.1.0.dev2.dist-info → salesforce_data_customcode-6.1.0.dev4.dist-info}/licenses/LICENSE.txt +0 -0
datacustomcode/io/reader/base.py
CHANGED
|
@@ -41,3 +41,45 @@ class BaseDataCloudReader(BaseDataAccessLayer):
|
|
|
41
41
|
name: str,
|
|
42
42
|
schema: Union[AtomicType, StructType, str, None] = None,
|
|
43
43
|
) -> PySparkDataFrame: ...
|
|
44
|
+
|
|
45
|
+
def read_dlo_deltas(self) -> PySparkDataFrame:
|
|
46
|
+
"""Read the streaming change feed (deltas) for a Data Lake Object.
|
|
47
|
+
|
|
48
|
+
This is the streaming counterpart to :meth:`read_dlo`. It returns a
|
|
49
|
+
streaming DataFrame over the change feed the Data Cloud runtime
|
|
50
|
+
publishes for a streaming (``DELTA_SYNC``) transform. Concrete
|
|
51
|
+
streaming behavior is provided by the deployed Data Cloud runtime; the
|
|
52
|
+
base implementation raises :class:`NotImplementedError` so local
|
|
53
|
+
readers that do not support streaming fail clearly.
|
|
54
|
+
|
|
55
|
+
Returns:
|
|
56
|
+
A streaming PySpark DataFrame over the DLO change feed.
|
|
57
|
+
|
|
58
|
+
Raises:
|
|
59
|
+
NotImplementedError: If the active reader does not support streaming
|
|
60
|
+
deltas (e.g. the local development readers).
|
|
61
|
+
"""
|
|
62
|
+
raise NotImplementedError(
|
|
63
|
+
"read_dlo_deltas is only supported when running in the Data Cloud "
|
|
64
|
+
"streaming runtime; the local reader does not support streaming "
|
|
65
|
+
"deltas."
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
def read_dmo_deltas(self) -> PySparkDataFrame:
|
|
69
|
+
"""Read the streaming change feed (deltas) for a Data Model Object.
|
|
70
|
+
|
|
71
|
+
Streaming counterpart to :meth:`read_dmo`. See :meth:`read_dlo_deltas`
|
|
72
|
+
for behavior and the local-development caveat.
|
|
73
|
+
|
|
74
|
+
Returns:
|
|
75
|
+
A streaming PySpark DataFrame over the DMO change feed.
|
|
76
|
+
|
|
77
|
+
Raises:
|
|
78
|
+
NotImplementedError: If the active reader does not support streaming
|
|
79
|
+
deltas (e.g. the local development readers).
|
|
80
|
+
"""
|
|
81
|
+
raise NotImplementedError(
|
|
82
|
+
"read_dmo_deltas is only supported when running in the Data Cloud "
|
|
83
|
+
"streaming runtime; the local reader does not support streaming "
|
|
84
|
+
"deltas."
|
|
85
|
+
)
|
datacustomcode/io/writer/base.py
CHANGED
|
@@ -22,6 +22,7 @@ from datacustomcode.io.base import BaseDataAccessLayer
|
|
|
22
22
|
|
|
23
23
|
if TYPE_CHECKING:
|
|
24
24
|
from pyspark.sql import DataFrame as PySparkDataFrame, SparkSession
|
|
25
|
+
from pyspark.sql.streaming import StreamingQuery
|
|
25
26
|
|
|
26
27
|
|
|
27
28
|
class WriteMode(str, Enum):
|
|
@@ -57,3 +58,35 @@ class BaseDataCloudWriter(BaseDataAccessLayer):
|
|
|
57
58
|
def write_to_dmo(
|
|
58
59
|
self, name: str, dataframe: PySparkDataFrame, write_mode: WriteMode
|
|
59
60
|
) -> None: ...
|
|
61
|
+
|
|
62
|
+
def write_dlo_deltas(
|
|
63
|
+
self, name: str, dataframe: PySparkDataFrame
|
|
64
|
+
) -> StreamingQuery:
|
|
65
|
+
"""Write a streaming DataFrame of deltas to a Data Lake Object.
|
|
66
|
+
|
|
67
|
+
Streaming counterpart to :meth:`write_to_dlo`. Starts a streaming query
|
|
68
|
+
that writes each micro-batch to the target DLO via the Data Cloud
|
|
69
|
+
streaming sink and returns the resulting ``StreamingQuery`` handle. The
|
|
70
|
+
runtime owns the trigger and checkpoint location; callers pass only the
|
|
71
|
+
table name. Concrete streaming behavior is provided by the deployed
|
|
72
|
+
Data Cloud runtime; the base implementation raises
|
|
73
|
+
:class:`NotImplementedError`.
|
|
74
|
+
|
|
75
|
+
Args:
|
|
76
|
+
name: Target Data Lake Object name.
|
|
77
|
+
dataframe: Streaming PySpark DataFrame produced from a
|
|
78
|
+
``read_dlo_deltas`` / ``read_dmo_deltas`` source.
|
|
79
|
+
|
|
80
|
+
Returns:
|
|
81
|
+
The started ``StreamingQuery``; the caller drives its lifecycle
|
|
82
|
+
(typically ``query.awaitTermination()``).
|
|
83
|
+
|
|
84
|
+
Raises:
|
|
85
|
+
NotImplementedError: If the active writer does not support streaming
|
|
86
|
+
deltas (e.g. the local development writers).
|
|
87
|
+
"""
|
|
88
|
+
raise NotImplementedError(
|
|
89
|
+
"write_dlo_deltas is only supported when running in the Data Cloud "
|
|
90
|
+
"streaming runtime; the local writer does not support streaming "
|
|
91
|
+
"deltas."
|
|
92
|
+
)
|
|
@@ -15,6 +15,7 @@
|
|
|
15
15
|
|
|
16
16
|
from datacustomcode.named_credential.base import NamedCredential
|
|
17
17
|
from datacustomcode.named_credential.default import DefaultNamedCredential
|
|
18
|
+
from datacustomcode.named_credential.errors import NamedCredentialCallError
|
|
18
19
|
from datacustomcode.named_credential.spark_base import SparkNamedCredential
|
|
19
20
|
from datacustomcode.named_credential.spark_default import DefaultSparkNamedCredential
|
|
20
21
|
|
|
@@ -22,5 +23,6 @@ __all__ = [
|
|
|
22
23
|
"DefaultNamedCredential",
|
|
23
24
|
"DefaultSparkNamedCredential",
|
|
24
25
|
"NamedCredential",
|
|
26
|
+
"NamedCredentialCallError",
|
|
25
27
|
"SparkNamedCredential",
|
|
26
28
|
]
|
|
@@ -31,11 +31,9 @@ from typing import (
|
|
|
31
31
|
import requests
|
|
32
32
|
|
|
33
33
|
from datacustomcode.named_credential.direct.auth import DynamicAuthHandler
|
|
34
|
-
from datacustomcode.named_credential.direct.credentials import
|
|
35
|
-
CredentialError,
|
|
36
|
-
CredentialStore,
|
|
37
|
-
)
|
|
34
|
+
from datacustomcode.named_credential.direct.credentials import CredentialStore
|
|
38
35
|
from datacustomcode.named_credential.direct.url_resolver import resolve_base_url
|
|
36
|
+
from datacustomcode.named_credential.errors import NamedCredentialCallError
|
|
39
37
|
from datacustomcode.token_provider import (
|
|
40
38
|
CredentialsTokenProvider,
|
|
41
39
|
SFCLITokenProvider,
|
|
@@ -69,7 +67,7 @@ class DirectCalloutTransport:
|
|
|
69
67
|
def callout(self, callout_request: Dict[str, Any]) -> Dict[str, Any]:
|
|
70
68
|
raw_url = callout_request["path"]
|
|
71
69
|
if not raw_url.startswith("callout:"):
|
|
72
|
-
raise
|
|
70
|
+
raise NamedCredentialCallError(
|
|
73
71
|
f"Callout URL must start with 'callout:', got '{raw_url}'."
|
|
74
72
|
)
|
|
75
73
|
|
|
@@ -83,7 +81,9 @@ class DirectCalloutTransport:
|
|
|
83
81
|
path_suffix = raw_url[sep_idx:]
|
|
84
82
|
|
|
85
83
|
if callout_key == "callout:":
|
|
86
|
-
raise
|
|
84
|
+
raise NamedCredentialCallError(
|
|
85
|
+
f"Named Credential name is empty in URL '{raw_url}'."
|
|
86
|
+
)
|
|
87
87
|
|
|
88
88
|
cred_config = self._store.get(callout_key)
|
|
89
89
|
base_url = self._base_url_cache.get(callout_key)
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# Copyright (c) 2025, Salesforce, Inc.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2
|
|
3
|
+
#
|
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
# you may not use this file except in compliance with the License.
|
|
6
|
+
# You may obtain a copy of the License at
|
|
7
|
+
#
|
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
#
|
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
# See the License for the specific language governing permissions and
|
|
14
|
+
# limitations under the License.
|
|
15
|
+
"""Exceptions raised by Named Credential implementations."""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
from typing import Optional
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class NamedCredentialCallError(RuntimeError):
|
|
23
|
+
"""Raised when a Named Credential external callout fails."""
|
|
24
|
+
|
|
25
|
+
def __init__(
|
|
26
|
+
self,
|
|
27
|
+
message: str,
|
|
28
|
+
*,
|
|
29
|
+
status: Optional[object] = None,
|
|
30
|
+
error_code: Optional[str] = None,
|
|
31
|
+
error_message: Optional[str] = None,
|
|
32
|
+
) -> None:
|
|
33
|
+
super().__init__(message)
|
|
34
|
+
self.status = status
|
|
35
|
+
self.error_code = error_code
|
|
36
|
+
self.error_message = error_message
|
datacustomcode/run.py
CHANGED
|
@@ -43,6 +43,15 @@ def _set_config_option(config_obj, key: str, value: Optional[str]) -> None:
|
|
|
43
43
|
config_obj.options[key] = value
|
|
44
44
|
|
|
45
45
|
|
|
46
|
+
def _read_streaming_source(config_json: dict) -> Optional[str]:
|
|
47
|
+
"""Return the streaming source name from config.json's ``streamingSource``."""
|
|
48
|
+
source = config_json.get("streamingSource")
|
|
49
|
+
if not isinstance(source, dict):
|
|
50
|
+
return None
|
|
51
|
+
name = source.get("name")
|
|
52
|
+
return str(name) if name else None
|
|
53
|
+
|
|
54
|
+
|
|
46
55
|
def _update_config_options(profile: Optional[str], sf_cli_org: Optional[str]):
|
|
47
56
|
if sf_cli_org:
|
|
48
57
|
config_key = "sf_cli_org"
|
|
@@ -132,6 +141,8 @@ def run_entrypoint(
|
|
|
132
141
|
_set_config_option(config.reader_config, "dataspace", dataspace)
|
|
133
142
|
_set_config_option(config.writer_config, "dataspace", dataspace)
|
|
134
143
|
|
|
144
|
+
config.streaming_source = _read_streaming_source(config_json)
|
|
145
|
+
|
|
135
146
|
_update_config_options(profile, sf_cli_org)
|
|
136
147
|
|
|
137
148
|
for dependency in dependencies:
|
datacustomcode/scan.py
CHANGED
|
@@ -15,6 +15,7 @@
|
|
|
15
15
|
from __future__ import annotations
|
|
16
16
|
|
|
17
17
|
import ast
|
|
18
|
+
import copy
|
|
18
19
|
import json
|
|
19
20
|
import os
|
|
20
21
|
import sys
|
|
@@ -32,6 +33,8 @@ import pydantic
|
|
|
32
33
|
from datacustomcode.version import get_version
|
|
33
34
|
|
|
34
35
|
DATA_ACCESS_METHODS = ["read_dlo", "read_dmo", "write_to_dlo", "write_to_dmo"]
|
|
36
|
+
STREAMING_READ_METHODS = ["read_dlo_deltas", "read_dmo_deltas"]
|
|
37
|
+
STREAMING_WRITE_METHODS = ["write_dlo_deltas"]
|
|
35
38
|
|
|
36
39
|
DATA_TRANSFORM_CONFIG_TEMPLATE = {
|
|
37
40
|
"sdkVersion": get_version(),
|
|
@@ -43,6 +46,20 @@ DATA_TRANSFORM_CONFIG_TEMPLATE = {
|
|
|
43
46
|
},
|
|
44
47
|
}
|
|
45
48
|
|
|
49
|
+
STREAMING_TRANSFORM_CONFIG_TEMPLATE = {
|
|
50
|
+
"sdkVersion": get_version(),
|
|
51
|
+
"entryPoint": "",
|
|
52
|
+
"dataspace": "default",
|
|
53
|
+
"streamingSource": {
|
|
54
|
+
"type": "dlo",
|
|
55
|
+
"name": "",
|
|
56
|
+
},
|
|
57
|
+
"permissions": {
|
|
58
|
+
"read": {},
|
|
59
|
+
"write": {},
|
|
60
|
+
},
|
|
61
|
+
}
|
|
62
|
+
|
|
46
63
|
FUNCTION_CONFIG_TEMPLATE = {
|
|
47
64
|
"entryPoint": "",
|
|
48
65
|
}
|
|
@@ -160,6 +177,35 @@ class DataAccessLayerCalls(pydantic.BaseModel):
|
|
|
160
177
|
return next(iter(self.write_to_dmo))
|
|
161
178
|
|
|
162
179
|
|
|
180
|
+
class StreamingDataAccessLayerCalls(pydantic.BaseModel):
|
|
181
|
+
read_dlo_deltas: bool
|
|
182
|
+
read_dmo_deltas: bool
|
|
183
|
+
write_dlo_deltas: frozenset[str]
|
|
184
|
+
|
|
185
|
+
@pydantic.model_validator(mode="after")
|
|
186
|
+
def validate_access_layer(self) -> StreamingDataAccessLayerCalls:
|
|
187
|
+
if self.read_dlo_deltas and self.read_dmo_deltas:
|
|
188
|
+
raise ValueError(
|
|
189
|
+
"Cannot read DLO and DMO deltas in the same streaming transform."
|
|
190
|
+
)
|
|
191
|
+
if not self.read_dlo_deltas and not self.read_dmo_deltas:
|
|
192
|
+
raise ValueError(
|
|
193
|
+
"A streaming transform must read from at least one DLO or DMO "
|
|
194
|
+
"delta stream (read_dlo_deltas / read_dmo_deltas)."
|
|
195
|
+
)
|
|
196
|
+
if not self.write_dlo_deltas:
|
|
197
|
+
raise ValueError(
|
|
198
|
+
"A streaming transform must write to at least one DLO via "
|
|
199
|
+
"write_dlo_deltas."
|
|
200
|
+
)
|
|
201
|
+
return self
|
|
202
|
+
|
|
203
|
+
@property
|
|
204
|
+
def read_layer(self) -> str:
|
|
205
|
+
"""Return the read source layer, ``"dlo"`` or ``"dmo"``."""
|
|
206
|
+
return "dlo" if self.read_dlo_deltas else "dmo"
|
|
207
|
+
|
|
208
|
+
|
|
163
209
|
class ClientMethodVisitor(ast.NodeVisitor):
|
|
164
210
|
"""AST Visitor that finds all instances of Client read/write method calls."""
|
|
165
211
|
|
|
@@ -168,6 +214,9 @@ class ClientMethodVisitor(ast.NodeVisitor):
|
|
|
168
214
|
self._read_dmo_instances: set[str] = set()
|
|
169
215
|
self._write_to_dlo_instances: set[str] = set()
|
|
170
216
|
self._write_to_dmo_instances: set[str] = set()
|
|
217
|
+
self._read_dlo_deltas: bool = False
|
|
218
|
+
self._read_dmo_deltas: bool = False
|
|
219
|
+
self._write_dlo_deltas_instances: set[str] = set()
|
|
171
220
|
self.variable_values: Dict[str, Union[str, None]] = {}
|
|
172
221
|
|
|
173
222
|
def visit_Assign(self, node: ast.Assign) -> None:
|
|
@@ -189,14 +238,15 @@ class ClientMethodVisitor(ast.NodeVisitor):
|
|
|
189
238
|
node.func.value, ast.Name
|
|
190
239
|
):
|
|
191
240
|
method_name = node.func.attr
|
|
241
|
+
|
|
242
|
+
if method_name == "read_dlo_deltas":
|
|
243
|
+
self._read_dlo_deltas = True
|
|
244
|
+
elif method_name == "read_dmo_deltas":
|
|
245
|
+
self._read_dmo_deltas = True
|
|
246
|
+
|
|
192
247
|
if method_name in DATA_ACCESS_METHODS and node.args:
|
|
193
248
|
arg = node.args[0]
|
|
194
|
-
name =
|
|
195
|
-
|
|
196
|
-
if isinstance(arg, ast.Constant) and isinstance(arg.value, str):
|
|
197
|
-
name = arg.value
|
|
198
|
-
elif isinstance(arg, ast.Name) and arg.id in self.variable_values:
|
|
199
|
-
name = self.variable_values[arg.id]
|
|
249
|
+
name = self._resolve_name_arg(arg)
|
|
200
250
|
|
|
201
251
|
if name:
|
|
202
252
|
if method_name == "read_dlo":
|
|
@@ -207,8 +257,29 @@ class ClientMethodVisitor(ast.NodeVisitor):
|
|
|
207
257
|
self._write_to_dlo_instances.add(name)
|
|
208
258
|
elif method_name == "write_to_dmo":
|
|
209
259
|
self._write_to_dmo_instances.add(name)
|
|
260
|
+
elif method_name in STREAMING_WRITE_METHODS and node.args:
|
|
261
|
+
name = self._resolve_name_arg(node.args[0])
|
|
262
|
+
if name and method_name == "write_dlo_deltas":
|
|
263
|
+
self._write_dlo_deltas_instances.add(name)
|
|
210
264
|
self.generic_visit(node)
|
|
211
265
|
|
|
266
|
+
def _resolve_name_arg(self, arg: ast.expr) -> Union[str, None]:
|
|
267
|
+
"""Resolve a string-literal or tracked-variable first argument."""
|
|
268
|
+
if isinstance(arg, ast.Constant) and isinstance(arg.value, str):
|
|
269
|
+
return arg.value
|
|
270
|
+
if isinstance(arg, ast.Name) and arg.id in self.variable_values:
|
|
271
|
+
return self.variable_values[arg.id]
|
|
272
|
+
return None
|
|
273
|
+
|
|
274
|
+
@property
|
|
275
|
+
def is_streaming(self) -> bool:
|
|
276
|
+
"""Whether any streaming (delta) access method was found."""
|
|
277
|
+
return (
|
|
278
|
+
self._read_dlo_deltas
|
|
279
|
+
or self._read_dmo_deltas
|
|
280
|
+
or bool(self._write_dlo_deltas_instances)
|
|
281
|
+
)
|
|
282
|
+
|
|
212
283
|
def found(self) -> DataAccessLayerCalls:
|
|
213
284
|
return DataAccessLayerCalls(
|
|
214
285
|
read_dlo=frozenset(self._read_dlo_instances),
|
|
@@ -217,6 +288,13 @@ class ClientMethodVisitor(ast.NodeVisitor):
|
|
|
217
288
|
write_to_dmo=frozenset(self._write_to_dmo_instances),
|
|
218
289
|
)
|
|
219
290
|
|
|
291
|
+
def found_streaming(self) -> StreamingDataAccessLayerCalls:
|
|
292
|
+
return StreamingDataAccessLayerCalls(
|
|
293
|
+
read_dlo_deltas=self._read_dlo_deltas,
|
|
294
|
+
read_dmo_deltas=self._read_dmo_deltas,
|
|
295
|
+
write_dlo_deltas=frozenset(self._write_dlo_deltas_instances),
|
|
296
|
+
)
|
|
297
|
+
|
|
220
298
|
|
|
221
299
|
class ImportVisitor(ast.NodeVisitor):
|
|
222
300
|
"""AST Visitor that extracts external package imports from Python code."""
|
|
@@ -301,23 +379,51 @@ def write_requirements_file(file_path: str) -> str:
|
|
|
301
379
|
return requirements_path
|
|
302
380
|
|
|
303
381
|
|
|
304
|
-
def
|
|
305
|
-
"""
|
|
382
|
+
def _visit_file(file_path: str) -> ClientMethodVisitor:
|
|
383
|
+
"""Parse a Python file and return the populated method visitor."""
|
|
306
384
|
with open(file_path, "r") as f:
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
385
|
+
tree = ast.parse(f.read())
|
|
386
|
+
visitor = ClientMethodVisitor()
|
|
387
|
+
visitor.visit(tree)
|
|
388
|
+
return visitor
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
def scan_file(file_path: str) -> DataAccessLayerCalls:
|
|
392
|
+
"""Scan a single Python file for batch Client read/write method calls."""
|
|
393
|
+
return _visit_file(file_path).found()
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def scan_file_streaming(file_path: str) -> StreamingDataAccessLayerCalls:
|
|
397
|
+
"""Scan a single Python file for StreamingClient delta method calls."""
|
|
398
|
+
return _visit_file(file_path).found_streaming()
|
|
399
|
+
|
|
312
400
|
|
|
401
|
+
def file_is_streaming(file_path: str) -> bool:
|
|
402
|
+
"""Return whether the entrypoint uses streaming (delta) access methods."""
|
|
403
|
+
return _visit_file(file_path).is_streaming
|
|
313
404
|
|
|
314
|
-
|
|
315
|
-
|
|
405
|
+
|
|
406
|
+
def dc_config_json_from_file(
|
|
407
|
+
file_path: str, type: str, streaming: bool = False
|
|
408
|
+
) -> dict[str, Any]:
|
|
409
|
+
"""Create a Data Cloud Custom Code config JSON from a script.
|
|
410
|
+
|
|
411
|
+
Args:
|
|
412
|
+
file_path: Path to the entrypoint.
|
|
413
|
+
type: Package type, ``"script"`` or ``"function"``.
|
|
414
|
+
streaming: For scripts, a streaming
|
|
415
|
+
(``streamingSource``) config instead of a batch one.
|
|
416
|
+
"""
|
|
316
417
|
config: dict[str, Any]
|
|
317
418
|
if type == "script":
|
|
318
|
-
|
|
419
|
+
template = (
|
|
420
|
+
STREAMING_TRANSFORM_CONFIG_TEMPLATE
|
|
421
|
+
if streaming
|
|
422
|
+
else DATA_TRANSFORM_CONFIG_TEMPLATE
|
|
423
|
+
)
|
|
424
|
+
config = copy.deepcopy(template)
|
|
319
425
|
elif type == "function":
|
|
320
|
-
config =
|
|
426
|
+
config = copy.deepcopy(FUNCTION_CONFIG_TEMPLATE)
|
|
321
427
|
config["entryPoint"] = os.path.basename(file_path)
|
|
322
428
|
return config
|
|
323
429
|
|
|
@@ -372,22 +478,51 @@ def update_config(file_path: str) -> dict[str, Any]:
|
|
|
372
478
|
|
|
373
479
|
if package_type == "script":
|
|
374
480
|
existing_config["dataspace"] = get_dataspace(existing_config)
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
if output.read_dlo:
|
|
378
|
-
read["dlo"] = list(output.read_dlo)
|
|
379
|
-
else:
|
|
380
|
-
read["dmo"] = list(output.read_dmo)
|
|
381
|
-
write: dict[str, list[str]] = {}
|
|
382
|
-
if output.write_to_dlo:
|
|
383
|
-
write["dlo"] = list(output.write_to_dlo)
|
|
481
|
+
if file_is_streaming(file_path):
|
|
482
|
+
_update_streaming_config(existing_config, file_path)
|
|
384
483
|
else:
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
484
|
+
existing_config.pop("streamingSource", None)
|
|
485
|
+
output = scan_file(file_path)
|
|
486
|
+
read: dict[str, list[str]] = {}
|
|
487
|
+
if output.read_dlo:
|
|
488
|
+
read["dlo"] = list(output.read_dlo)
|
|
489
|
+
else:
|
|
490
|
+
read["dmo"] = list(output.read_dmo)
|
|
491
|
+
write: dict[str, list[str]] = {}
|
|
492
|
+
if output.write_to_dlo:
|
|
493
|
+
write["dlo"] = list(output.write_to_dlo)
|
|
494
|
+
else:
|
|
495
|
+
write["dmo"] = list(output.write_to_dmo)
|
|
496
|
+
|
|
497
|
+
existing_config["permissions"] = {"read": read, "write": write}
|
|
388
498
|
return existing_config
|
|
389
499
|
|
|
390
500
|
|
|
501
|
+
def _update_streaming_config(existing_config: dict[str, Any], file_path: str) -> None:
|
|
502
|
+
output = scan_file_streaming(file_path)
|
|
503
|
+
read_layer = output.read_layer
|
|
504
|
+
|
|
505
|
+
source = existing_config.get("streamingSource")
|
|
506
|
+
if not isinstance(source, dict):
|
|
507
|
+
source = {}
|
|
508
|
+
source_name = source.get("name", "")
|
|
509
|
+
existing_config["streamingSource"] = {"type": read_layer, "name": source_name}
|
|
510
|
+
|
|
511
|
+
if not source_name:
|
|
512
|
+
logger.warning(
|
|
513
|
+
"streamingSource.name is empty in config.json. A streaming "
|
|
514
|
+
"transform must declare its read source; set streamingSource.name "
|
|
515
|
+
"to the DLO/DMO the transform reads from."
|
|
516
|
+
)
|
|
517
|
+
|
|
518
|
+
read_names = [source_name] if source_name else []
|
|
519
|
+
write_names = list(output.write_dlo_deltas)
|
|
520
|
+
existing_config["permissions"] = {
|
|
521
|
+
"read": {read_layer: read_names},
|
|
522
|
+
"write": {"dlo": write_names},
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
|
|
391
526
|
def get_dataspace(existing_config: dict[str, str]) -> str:
|
|
392
527
|
if "dataspace" in existing_config:
|
|
393
528
|
dataspace_value = existing_config["dataspace"]
|
datacustomcode/template.py
CHANGED
|
@@ -23,8 +23,12 @@ from datacustomcode.constants import FEATURE_TEMPLATE_MAPPING
|
|
|
23
23
|
script_template_dir = os.path.join(os.path.dirname(__file__), "templates", "script")
|
|
24
24
|
function_template_dir = os.path.join(os.path.dirname(__file__), "templates", "function")
|
|
25
25
|
|
|
26
|
+
STREAMING_EXAMPLE_ENTRYPOINT = os.path.join(
|
|
27
|
+
script_template_dir, "examples", "streaming_deltas", "entrypoint.py"
|
|
28
|
+
)
|
|
26
29
|
|
|
27
|
-
|
|
30
|
+
|
|
31
|
+
def copy_script_template(target_dir: str, streaming: bool = False) -> None:
|
|
28
32
|
"""Copy the template to the target directory."""
|
|
29
33
|
os.makedirs(target_dir, exist_ok=True)
|
|
30
34
|
|
|
@@ -39,6 +43,14 @@ def copy_script_template(target_dir: str) -> None:
|
|
|
39
43
|
logger.debug(f"Copying file {source} to {destination}...")
|
|
40
44
|
shutil.copy2(source, destination)
|
|
41
45
|
|
|
46
|
+
if streaming:
|
|
47
|
+
destination = os.path.join(target_dir, "payload", "entrypoint.py")
|
|
48
|
+
logger.debug(
|
|
49
|
+
f"Copying streaming example {STREAMING_EXAMPLE_ENTRYPOINT} to "
|
|
50
|
+
f"{destination}..."
|
|
51
|
+
)
|
|
52
|
+
shutil.copy2(STREAMING_EXAMPLE_ENTRYPOINT, destination)
|
|
53
|
+
|
|
42
54
|
|
|
43
55
|
def copy_function_template(target_dir: str, use_in_feature: Optional[str]) -> None:
|
|
44
56
|
os.makedirs(target_dir, exist_ok=True)
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""Streaming BYOC transform: read a DLO change feed and write the deltas back.
|
|
2
|
+
|
|
3
|
+
This example is the streaming counterpart to a normal batch entrypoint. Instead
|
|
4
|
+
of a batch ``Client`` with ``read_dlo`` / ``write_to_dlo`` (which read and write
|
|
5
|
+
a bounded snapshot), it uses a :class:`StreamingClient` and its streaming delta
|
|
6
|
+
methods:
|
|
7
|
+
|
|
8
|
+
* ``client.read_dlo_deltas()`` returns a *streaming* DataFrame over the
|
|
9
|
+
Change Data Feed of the source DLO. Each row carries the source columns plus
|
|
10
|
+
change-feed metadata columns (``_record_type``, ``_commit_*``).
|
|
11
|
+
* ``client.write_dlo_deltas(name, df)`` starts a streaming query that writes
|
|
12
|
+
each micro-batch to the target DLO and returns the ``StreamingQuery`` handle.
|
|
13
|
+
The runtime owns the trigger, and checkpoint location — the caller only
|
|
14
|
+
chooses the table.
|
|
15
|
+
|
|
16
|
+
The transform in between is ordinary PySpark. Because the source is a change
|
|
17
|
+
feed, keep the metadata columns on the DataFrame you hand to
|
|
18
|
+
``write_dlo_deltas`` — the sink relies on them to merge changes correctly.
|
|
19
|
+
|
|
20
|
+
This entrypoint only runs inside the Data Cloud streaming (``DELTA_SYNC``)
|
|
21
|
+
runtime; the local ``datacustomcode run`` readers/writers raise
|
|
22
|
+
``NotImplementedError`` for the delta methods.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
from pyspark.sql.functions import col, upper
|
|
26
|
+
|
|
27
|
+
from datacustomcode.client import StreamingClient
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def main():
|
|
31
|
+
client = StreamingClient()
|
|
32
|
+
|
|
33
|
+
# Streaming DataFrame over the source DLO's change feed.
|
|
34
|
+
deltas = client.read_dlo_deltas()
|
|
35
|
+
|
|
36
|
+
# Ordinary PySpark transform.
|
|
37
|
+
transformed = deltas.withColumn("description__c", upper(col("description__c")))
|
|
38
|
+
|
|
39
|
+
# Start the streaming write. write_dlo_deltas returns the StreamingQuery;
|
|
40
|
+
# the trigger and checkpoint location are provided by the runtime.
|
|
41
|
+
query = client.write_dlo_deltas("Account_std_copy__dll", transformed)
|
|
42
|
+
|
|
43
|
+
# Drive the query's lifecycle. In the streaming runtime this blocks until
|
|
44
|
+
# the job is stopped by the platform.
|
|
45
|
+
query.awaitTermination()
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
if __name__ == "__main__":
|
|
49
|
+
main()
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: salesforce-data-customcode
|
|
3
|
-
Version: 6.1.0.
|
|
3
|
+
Version: 6.1.0.dev4
|
|
4
4
|
Summary: Data Cloud Custom Code SDK
|
|
5
5
|
License-Expression: Apache-2.0
|
|
6
6
|
License-File: LICENSE.txt
|
|
@@ -170,15 +170,22 @@ Your Python dependencies can be packaged as .py files, .zip archives (containing
|
|
|
170
170
|
|
|
171
171
|
## API
|
|
172
172
|
|
|
173
|
-
Your entry point script will define logic using the `Client` object which
|
|
173
|
+
Your entry point script will define logic using the `Client` object (for batch transforms) or the `StreamingClient` object (for streaming delta transforms), which wrap the data access layers. Both are singletons; a single transform should use one or the other, not both.
|
|
174
174
|
|
|
175
|
-
You should only need the following methods:
|
|
175
|
+
For a batch transform, use `Client`. You should only need the following methods:
|
|
176
176
|
* `find_file_path(file_name)` – Resolve a bundled file (placed under `payload/files/`) to a `pathlib.Path` that exists. Works the same locally and inside Data Cloud — see [Bundled file resolution](#bundled-file-resolution) below for the full lookup order. Raises `FileNotFoundError` if the file isn't found.
|
|
177
177
|
* `read_dlo(name)` – Read from a Data Lake Object by name
|
|
178
178
|
* `read_dmo(name)` – Read from a Data Model Object by name
|
|
179
179
|
* `write_to_dlo(name, spark_dataframe, write_mode)` – Write to a Data Model Object by name with a Spark dataframe
|
|
180
180
|
* `write_to_dmo(name, spark_dataframe, write_mode)` – Write to a Data Lake Object by name with a Spark dataframe
|
|
181
181
|
|
|
182
|
+
For a streaming (delta) transform, use `StreamingClient`, which exposes the streaming counterparts:
|
|
183
|
+
* `read_dlo_deltas()` – Read the streaming change feed (deltas) of a Data Lake Object as a streaming DataFrame.
|
|
184
|
+
* `read_dmo_deltas()` – Read the streaming change feed (deltas) of a Data Model Object as a streaming DataFrame.
|
|
185
|
+
* `write_dlo_deltas(name, spark_dataframe)` – Write a streaming DataFrame of deltas to a Data Lake Object; returns the started `StreamingQuery`
|
|
186
|
+
|
|
187
|
+
`find_file_path`, `llm_gateway_generate_text`, and `einstein_predict` are available on both clients.
|
|
188
|
+
|
|
182
189
|
For example:
|
|
183
190
|
```python
|
|
184
191
|
from datacustomcode import Client
|
|
@@ -194,6 +201,36 @@ client.write_to_dlo('output_DLO')
|
|
|
194
201
|
> [!WARNING]
|
|
195
202
|
> Currently we only support reading from DMOs and writing to DMOs or reading from DLOs and writing to DLOs, but they cannot mix.
|
|
196
203
|
|
|
204
|
+
### Streaming (delta) transforms
|
|
205
|
+
|
|
206
|
+
Streaming BYOC transforms process a Data Lake Object's Change Data Feed continuously instead of reading a bounded snapshot. Use a `StreamingClient` and its `*_deltas` methods in place of the batch `Client` read/write methods:
|
|
207
|
+
|
|
208
|
+
```python
|
|
209
|
+
from pyspark.sql.functions import col, upper
|
|
210
|
+
|
|
211
|
+
from datacustomcode import StreamingClient
|
|
212
|
+
|
|
213
|
+
client = StreamingClient()
|
|
214
|
+
|
|
215
|
+
# read_dlo_deltas returns a *streaming* DataFrame over the change feed.
|
|
216
|
+
# The runtime resolves the single streaming source, so no name is passed.
|
|
217
|
+
deltas = client.read_dlo_deltas()
|
|
218
|
+
|
|
219
|
+
# Ordinary PySpark transform.
|
|
220
|
+
transformed = deltas.withColumn("description__c", upper(col("description__c")))
|
|
221
|
+
|
|
222
|
+
# write_dlo_deltas starts a streaming query and returns the StreamingQuery.
|
|
223
|
+
# The runtime owns the trigger and checkpoint location; you
|
|
224
|
+
# choose only the target table.
|
|
225
|
+
query = client.write_dlo_deltas("Output__dll", transformed)
|
|
226
|
+
query.awaitTermination()
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
Notes:
|
|
230
|
+
|
|
231
|
+
- These methods only run inside the Data Cloud streaming (`DELTA_SYNC`) runtime. Locally (`datacustomcode run`) they raise `NotImplementedError`, since there is no change feed to stream.
|
|
232
|
+
- A complete runnable entry point is provided in [`examples/streaming_deltas/entrypoint.py`](src/datacustomcode/templates/script/examples/streaming_deltas/entrypoint.py).
|
|
233
|
+
|
|
197
234
|
### Bundled file resolution
|
|
198
235
|
|
|
199
236
|
Place bundled files (CSVs, prompt files, etc.) under `payload/files/`. The same `client.find_file_path("data.csv")` call resolves consistently across all three runtimes:
|