salesforce-data-customcode 6.0.5.dev1__py3-none-any.whl → 6.0.6.dev2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- datacustomcode/__init__.py +20 -0
- datacustomcode/client.py +325 -98
- datacustomcode/config.py +5 -0
- datacustomcode/config.yaml +3 -0
- datacustomcode/einstein_predictions/__init__.py +8 -0
- datacustomcode/einstein_predictions/errors.py +36 -0
- datacustomcode/einstein_predictions/spark_base.py +72 -0
- datacustomcode/einstein_predictions/spark_default.py +239 -0
- datacustomcode/einstein_predictions_config.py +45 -1
- datacustomcode/io/reader/base.py +42 -0
- datacustomcode/io/writer/base.py +33 -0
- datacustomcode/run.py +11 -0
- datacustomcode/templates/script/examples/streaming_deltas/entrypoint.py +49 -0
- datacustomcode/templates/script/payload/entrypoint.py +17 -0
- {salesforce_data_customcode-6.0.5.dev1.dist-info → salesforce_data_customcode-6.0.6.dev2.dist-info}/METADATA +96 -8
- {salesforce_data_customcode-6.0.5.dev1.dist-info → salesforce_data_customcode-6.0.6.dev2.dist-info}/RECORD +19 -15
- {salesforce_data_customcode-6.0.5.dev1.dist-info → salesforce_data_customcode-6.0.6.dev2.dist-info}/WHEEL +0 -0
- {salesforce_data_customcode-6.0.5.dev1.dist-info → salesforce_data_customcode-6.0.6.dev2.dist-info}/entry_points.txt +0 -0
- {salesforce_data_customcode-6.0.5.dev1.dist-info → salesforce_data_customcode-6.0.6.dev2.dist-info}/licenses/LICENSE.txt +0 -0
datacustomcode/__init__.py
CHANGED
|
@@ -17,10 +17,14 @@ __all__ = [
|
|
|
17
17
|
"AuthType",
|
|
18
18
|
"Client",
|
|
19
19
|
"Credentials",
|
|
20
|
+
"DefaultSparkEinsteinPredictions",
|
|
20
21
|
"DefaultSparkLLMGateway",
|
|
21
22
|
"PrintDataCloudWriter",
|
|
22
23
|
"QueryAPIDataCloudReader",
|
|
24
|
+
"SparkEinsteinPredictions",
|
|
23
25
|
"SparkLLMGateway",
|
|
26
|
+
"StreamingClient",
|
|
27
|
+
"einstein_predict_col",
|
|
24
28
|
"llm_gateway_generate_text_col",
|
|
25
29
|
]
|
|
26
30
|
|
|
@@ -31,6 +35,10 @@ def __getattr__(name: str):
|
|
|
31
35
|
from datacustomcode.client import Client
|
|
32
36
|
|
|
33
37
|
return Client
|
|
38
|
+
elif name == "StreamingClient":
|
|
39
|
+
from datacustomcode.client import StreamingClient
|
|
40
|
+
|
|
41
|
+
return StreamingClient
|
|
34
42
|
elif name == "AuthType":
|
|
35
43
|
from datacustomcode.credentials import AuthType
|
|
36
44
|
|
|
@@ -59,4 +67,16 @@ def __getattr__(name: str):
|
|
|
59
67
|
from datacustomcode.client import llm_gateway_generate_text_col
|
|
60
68
|
|
|
61
69
|
return llm_gateway_generate_text_col
|
|
70
|
+
elif name == "SparkEinsteinPredictions":
|
|
71
|
+
from datacustomcode.einstein_predictions import SparkEinsteinPredictions
|
|
72
|
+
|
|
73
|
+
return SparkEinsteinPredictions
|
|
74
|
+
elif name == "DefaultSparkEinsteinPredictions":
|
|
75
|
+
from datacustomcode.einstein_predictions import DefaultSparkEinsteinPredictions
|
|
76
|
+
|
|
77
|
+
return DefaultSparkEinsteinPredictions
|
|
78
|
+
elif name == "einstein_predict_col":
|
|
79
|
+
from datacustomcode.client import einstein_predict_col
|
|
80
|
+
|
|
81
|
+
return einstein_predict_col
|
|
62
82
|
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
datacustomcode/client.py
CHANGED
|
@@ -17,13 +17,17 @@ from __future__ import annotations
|
|
|
17
17
|
from enum import Enum
|
|
18
18
|
from typing import (
|
|
19
19
|
TYPE_CHECKING,
|
|
20
|
+
Any,
|
|
20
21
|
ClassVar,
|
|
21
22
|
Dict,
|
|
22
23
|
Optional,
|
|
24
|
+
TypeVar,
|
|
23
25
|
Union,
|
|
26
|
+
cast,
|
|
24
27
|
)
|
|
25
28
|
|
|
26
29
|
from datacustomcode.config import config
|
|
30
|
+
from datacustomcode.einstein_predictions_config import spark_einstein_predictions_config
|
|
27
31
|
from datacustomcode.file.path.default import DefaultFindFilePath
|
|
28
32
|
from datacustomcode.io.reader.base import BaseDataCloudReader
|
|
29
33
|
from datacustomcode.llm_gateway_config import spark_llm_gateway_config
|
|
@@ -32,14 +36,55 @@ from datacustomcode.spark.default import DefaultSparkSessionProvider
|
|
|
32
36
|
if TYPE_CHECKING:
|
|
33
37
|
from pathlib import Path
|
|
34
38
|
|
|
35
|
-
from pyspark.sql import
|
|
39
|
+
from pyspark.sql import (
|
|
40
|
+
Column,
|
|
41
|
+
DataFrame as PySparkDataFrame,
|
|
42
|
+
SparkSession,
|
|
43
|
+
)
|
|
44
|
+
from pyspark.sql.streaming import StreamingQuery
|
|
36
45
|
|
|
46
|
+
from datacustomcode.einstein_predictions.spark_base import SparkEinsteinPredictions
|
|
47
|
+
from datacustomcode.einstein_predictions.types import PredictionType
|
|
37
48
|
from datacustomcode.io.reader.base import BaseDataCloudReader
|
|
38
49
|
from datacustomcode.io.writer.base import BaseDataCloudWriter, WriteMode
|
|
39
50
|
from datacustomcode.llm_gateway.spark_base import SparkLLMGateway
|
|
40
51
|
from datacustomcode.spark.base import BaseSparkSessionProvider
|
|
41
52
|
|
|
42
53
|
|
|
54
|
+
def _streaming_source_name() -> str:
|
|
55
|
+
"""Return the streaming transform's read-source name.
|
|
56
|
+
|
|
57
|
+
Resolved from ``config.streaming_source``, which ``run_entrypoint``
|
|
58
|
+
populates from config.json's ``streamingSource`` field.
|
|
59
|
+
|
|
60
|
+
Raises:
|
|
61
|
+
RuntimeError: If no ``streaming_source`` has been configured (e.g. the
|
|
62
|
+
transform's config.json has no ``streamingSource`` field).
|
|
63
|
+
"""
|
|
64
|
+
source = config.streaming_source
|
|
65
|
+
if not source:
|
|
66
|
+
raise RuntimeError(
|
|
67
|
+
"No streaming source configured. A streaming transform must declare "
|
|
68
|
+
"its read source in config.json under 'streamingSource'."
|
|
69
|
+
)
|
|
70
|
+
return source
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _active_client() -> "_BaseClient":
|
|
74
|
+
"""Return the client backing the module-level Spark column helpers.
|
|
75
|
+
|
|
76
|
+
Prefers an already-initialized singleton so a streaming job reuses its
|
|
77
|
+
:class:`StreamingClient` (and a batch job its :class:`Client`) rather than
|
|
78
|
+
forcing an unrelated client into existence. Falls back to building the
|
|
79
|
+
batch :class:`Client` when neither has been created yet.
|
|
80
|
+
"""
|
|
81
|
+
if Client._instance is not None:
|
|
82
|
+
return Client._instance
|
|
83
|
+
if StreamingClient._instance is not None:
|
|
84
|
+
return StreamingClient._instance
|
|
85
|
+
return Client()
|
|
86
|
+
|
|
87
|
+
|
|
43
88
|
def _build_spark_llm_gateway() -> "SparkLLMGateway":
|
|
44
89
|
"""Instantiate the SDK-configured :class:`SparkLLMGateway`.
|
|
45
90
|
|
|
@@ -95,10 +140,74 @@ def llm_gateway_generate_text_col(
|
|
|
95
140
|
the generated text; on failure, ``status == "ERROR"`` and the
|
|
96
141
|
``error_*`` fields carry diagnostic detail.
|
|
97
142
|
"""
|
|
98
|
-
gateway =
|
|
143
|
+
gateway = _active_client()._get_spark_llm_gateway()
|
|
99
144
|
return gateway.llm_gateway_generate_text_col(template, values, model_id=model_id)
|
|
100
145
|
|
|
101
146
|
|
|
147
|
+
def _build_spark_einstein_predictions() -> "SparkEinsteinPredictions":
|
|
148
|
+
"""Instantiate the SDK-configured :class:`SparkEinsteinPredictions`.
|
|
149
|
+
|
|
150
|
+
Raises:
|
|
151
|
+
RuntimeError: If no ``spark_einstein_predictions_config`` has been loaded.
|
|
152
|
+
"""
|
|
153
|
+
cfg = spark_einstein_predictions_config.spark_einstein_predictions_config
|
|
154
|
+
if cfg is None:
|
|
155
|
+
raise RuntimeError(
|
|
156
|
+
"spark_einstein_predictions_config is not configured. Add a "
|
|
157
|
+
"'spark_einstein_predictions_config' section to config.yaml."
|
|
158
|
+
)
|
|
159
|
+
return cfg.to_object()
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def einstein_predict_col(
|
|
163
|
+
model_api_name: str,
|
|
164
|
+
prediction_type: "PredictionType",
|
|
165
|
+
features: Dict[str, "Column"],
|
|
166
|
+
settings: Optional[Dict[str, Any]] = None,
|
|
167
|
+
) -> "Column":
|
|
168
|
+
"""Build a Spark Column that runs an Einstein prediction per row.
|
|
169
|
+
|
|
170
|
+
The returned Column yields a struct ``{status, response, error_code,
|
|
171
|
+
error_message}`` for each row. Use ``[...]`` (or ``getField``) to pick the
|
|
172
|
+
field you want, e.g. ``einstein_predict_col(...)["response"]``. ``response``
|
|
173
|
+
holds the prediction response payload as a JSON string. Per-row failures
|
|
174
|
+
populate ``status`` / ``error_code`` / ``error_message`` so a single bad row
|
|
175
|
+
does not abort the whole Spark job.
|
|
176
|
+
|
|
177
|
+
Example:
|
|
178
|
+
|
|
179
|
+
>>> from datacustomcode.einstein_predictions.types import PredictionType
|
|
180
|
+
>>> result = einstein_predict_col(
|
|
181
|
+
... "my_regression_model",
|
|
182
|
+
... PredictionType.REGRESSION,
|
|
183
|
+
... {"square_feet": col("square_feet__c"), "beds": col("beds__c")},
|
|
184
|
+
... )
|
|
185
|
+
>>> df.withColumn("prediction__c", result["response"])
|
|
186
|
+
>>> # …or keep the struct around and inspect failures:
|
|
187
|
+
>>> df.withColumn("pred", result).select(
|
|
188
|
+
... "pred.status", "pred.response", "pred.error_message"
|
|
189
|
+
... )
|
|
190
|
+
|
|
191
|
+
Args:
|
|
192
|
+
model_api_name: API name of the Einstein model to invoke.
|
|
193
|
+
prediction_type: The :class:`PredictionType` of the model.
|
|
194
|
+
features: A mapping from model feature column name to a Spark ``Column``
|
|
195
|
+
supplying that feature's per-row value.
|
|
196
|
+
settings: Optional prediction settings forwarded to the model.
|
|
197
|
+
|
|
198
|
+
Returns:
|
|
199
|
+
A Spark ``Column`` of ``StructType`` with fields ``status``,
|
|
200
|
+
``response``, ``error_code``, and ``error_message`` (all nullable
|
|
201
|
+
strings). On success, ``status == "SUCCESS"`` and ``response`` holds
|
|
202
|
+
the JSON-serialized prediction payload; on failure, ``status ==
|
|
203
|
+
"ERROR"`` and the ``error_*`` fields carry diagnostic detail.
|
|
204
|
+
"""
|
|
205
|
+
predictions = _active_client()._get_spark_einstein_predictions()
|
|
206
|
+
return predictions.einstein_predict_col(
|
|
207
|
+
model_api_name, prediction_type, features, settings=settings
|
|
208
|
+
)
|
|
209
|
+
|
|
210
|
+
|
|
102
211
|
class DataCloudObjectType(Enum):
|
|
103
212
|
DLO = "dlo"
|
|
104
213
|
DMO = "dmo"
|
|
@@ -137,74 +246,86 @@ class DataCloudAccessLayerException(Exception):
|
|
|
137
246
|
return msg
|
|
138
247
|
|
|
139
248
|
|
|
140
|
-
|
|
141
|
-
|
|
249
|
+
_ClientT = TypeVar("_ClientT", bound="_BaseClient")
|
|
250
|
+
|
|
142
251
|
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
can read from DLOs and write to DLOs, read from DMOs and write to DMOs, but you
|
|
146
|
-
cannot read from DLOs and write to DMOs or read from DMOs and write to DLOs.
|
|
147
|
-
Furthermore you cannot mix during merging tables. This class is a singleton to
|
|
148
|
-
prevent accidental mixing of DLOs and DMOs.
|
|
252
|
+
class _BaseClient:
|
|
253
|
+
"""Shared machinery for the Data Cloud client singletons.
|
|
149
254
|
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
255
|
+
Holds the wiring common to :class:`Client` (batch) and
|
|
256
|
+
:class:`StreamingClient`
|
|
257
|
+
|
|
258
|
+
This base class is not meant to be instantiated directly; use
|
|
259
|
+
:class:`Client` or :class:`StreamingClient`.
|
|
155
260
|
|
|
156
261
|
Args:
|
|
157
|
-
finder: Find a file path
|
|
158
262
|
reader: A custom reader to use for reading Data Cloud objects.
|
|
159
263
|
writer: A custom writer to use for writing Data Cloud objects.
|
|
264
|
+
spark_provider: Optional custom :class:`BaseSparkSessionProvider`.
|
|
160
265
|
spark_llm_gateway: Optional custom :class:`SparkLLMGateway`.
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
>>> client = Client()
|
|
164
|
-
>>> file_path = client.find_file_path("data.csv")
|
|
165
|
-
>>> dlo = client.read_dlo("my_dlo")
|
|
166
|
-
>>> client.write_to_dmo("my_dmo", dlo)
|
|
167
|
-
>>> answer = client.llm_gateway_generate_text("Generate a greeting message")
|
|
266
|
+
spark_einstein_predictions: Optional custom
|
|
267
|
+
:class:`SparkEinsteinPredictions`.
|
|
168
268
|
"""
|
|
169
269
|
|
|
170
|
-
_instance:
|
|
270
|
+
# Each concrete subclass gets its own ``_instance`` slot: reads fall through
|
|
271
|
+
# to this base default of ``None``, but ``cls._instance = ...`` in __new__
|
|
272
|
+
# always writes to the subclass, so ``Client`` and ``StreamingClient`` never
|
|
273
|
+
# share an instance.
|
|
274
|
+
_instance: ClassVar[Optional[_BaseClient]] = None
|
|
275
|
+
# Process-wide Spark session shared across BOTH client types. Unlike
|
|
276
|
+
# ``_instance``, this is written via ``_BaseClient._shared_spark`` (never
|
|
277
|
+
# ``cls._shared_spark``), so the slot lives on the base class and a
|
|
278
|
+
# ``Client`` and a ``StreamingClient`` in the same process reuse one session
|
|
279
|
+
# — and therefore one underlying connection — instead of opening two
|
|
280
|
+
# containing differing state
|
|
281
|
+
_shared_spark: ClassVar[Optional[SparkSession]] = None
|
|
171
282
|
_reader: BaseDataCloudReader
|
|
172
283
|
_writer: BaseDataCloudWriter
|
|
173
284
|
_file: DefaultFindFilePath
|
|
174
285
|
_spark_llm_gateway: Optional[SparkLLMGateway]
|
|
286
|
+
_spark_einstein_predictions: Optional[SparkEinsteinPredictions]
|
|
175
287
|
_data_layer_history: dict[DataCloudObjectType, set[str]]
|
|
176
288
|
_code_type: str
|
|
177
289
|
|
|
178
290
|
def __new__(
|
|
179
|
-
cls,
|
|
291
|
+
cls: type[_ClientT],
|
|
180
292
|
reader: Optional[BaseDataCloudReader] = None,
|
|
181
293
|
writer: Optional[BaseDataCloudWriter] = None,
|
|
182
294
|
spark_provider: Optional[BaseSparkSessionProvider] = None,
|
|
183
295
|
spark_llm_gateway: Optional[SparkLLMGateway] = None,
|
|
296
|
+
spark_einstein_predictions: Optional[SparkEinsteinPredictions] = None,
|
|
184
297
|
code_type: str = "script",
|
|
185
|
-
) ->
|
|
298
|
+
) -> _ClientT:
|
|
186
299
|
|
|
187
300
|
if cls._instance is None:
|
|
188
|
-
|
|
189
|
-
|
|
301
|
+
instance = super().__new__(cls)
|
|
302
|
+
instance._spark_llm_gateway = spark_llm_gateway
|
|
303
|
+
instance._spark_einstein_predictions = spark_einstein_predictions
|
|
190
304
|
# Initialize Readers and Writers from config
|
|
191
305
|
# and/or provided reader and writer
|
|
192
306
|
if reader is None or writer is None:
|
|
193
|
-
# We need a spark because we will initialize readers and writers
|
|
194
|
-
if
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
provider: BaseSparkSessionProvider
|
|
200
|
-
if spark_provider is not None:
|
|
201
|
-
provider = spark_provider
|
|
202
|
-
elif config.spark_provider_config is not None:
|
|
203
|
-
provider = config.spark_provider_config.to_object()
|
|
307
|
+
# We need a spark because we will initialize readers and writers.
|
|
308
|
+
# Reuse the process-wide session if one client already built it,
|
|
309
|
+
# so a Client and a StreamingClient share a single connection.
|
|
310
|
+
if _BaseClient._shared_spark is not None:
|
|
311
|
+
spark = _BaseClient._shared_spark
|
|
204
312
|
else:
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
313
|
+
if config.spark_config is None:
|
|
314
|
+
raise ValueError(
|
|
315
|
+
"Spark config is required when reader/writer is not "
|
|
316
|
+
"provided"
|
|
317
|
+
)
|
|
318
|
+
|
|
319
|
+
provider: BaseSparkSessionProvider
|
|
320
|
+
if spark_provider is not None:
|
|
321
|
+
provider = spark_provider
|
|
322
|
+
elif config.spark_provider_config is not None:
|
|
323
|
+
provider = config.spark_provider_config.to_object()
|
|
324
|
+
else:
|
|
325
|
+
provider = DefaultSparkSessionProvider()
|
|
326
|
+
|
|
327
|
+
spark = provider.get_session(config.spark_config)
|
|
328
|
+
_BaseClient._shared_spark = spark
|
|
208
329
|
|
|
209
330
|
if config.reader_config is None and reader is None:
|
|
210
331
|
raise ValueError(
|
|
@@ -227,66 +348,17 @@ class Client:
|
|
|
227
348
|
else:
|
|
228
349
|
writer_init = writer
|
|
229
350
|
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
351
|
+
instance._reader = reader_init
|
|
352
|
+
instance._writer = writer_init
|
|
353
|
+
instance._file = DefaultFindFilePath()
|
|
354
|
+
instance._data_layer_history = {
|
|
234
355
|
DataCloudObjectType.DLO: set(),
|
|
235
356
|
DataCloudObjectType.DMO: set(),
|
|
236
357
|
}
|
|
237
|
-
|
|
358
|
+
cls._instance = instance
|
|
359
|
+
elif reader is not None or writer is not None:
|
|
238
360
|
raise ValueError("Cannot set reader or writer after client is initialized")
|
|
239
|
-
return cls._instance
|
|
240
|
-
|
|
241
|
-
def read_dlo(self, name: str) -> PySparkDataFrame:
|
|
242
|
-
"""Read a DLO from Data Cloud.
|
|
243
|
-
|
|
244
|
-
Args:
|
|
245
|
-
name: The name of the DLO to read.
|
|
246
|
-
|
|
247
|
-
Returns:
|
|
248
|
-
A PySpark DataFrame containing the DLO data.
|
|
249
|
-
"""
|
|
250
|
-
self._record_dlo_access(name)
|
|
251
|
-
return self._reader.read_dlo(name) # type: ignore[no-any-return]
|
|
252
|
-
|
|
253
|
-
def read_dmo(self, name: str) -> PySparkDataFrame:
|
|
254
|
-
"""Read a DMO from Data Cloud.
|
|
255
|
-
|
|
256
|
-
Args:
|
|
257
|
-
name: The name of the DMO to read.
|
|
258
|
-
|
|
259
|
-
Returns:
|
|
260
|
-
A PySpark DataFrame containing the DMO data.
|
|
261
|
-
"""
|
|
262
|
-
self._record_dmo_access(name)
|
|
263
|
-
return self._reader.read_dmo(name) # type: ignore[no-any-return]
|
|
264
|
-
|
|
265
|
-
def write_to_dlo(
|
|
266
|
-
self, name: str, dataframe: PySparkDataFrame, write_mode: WriteMode, **kwargs
|
|
267
|
-
) -> None:
|
|
268
|
-
"""Write a PySpark DataFrame to a DLO in Data Cloud.
|
|
269
|
-
|
|
270
|
-
Args:
|
|
271
|
-
name: The name of the DLO to write to.
|
|
272
|
-
dataframe: The PySpark DataFrame to write.
|
|
273
|
-
write_mode: The write mode to use for writing to the DLO.
|
|
274
|
-
"""
|
|
275
|
-
self._validate_data_layer_history_does_not_contain(DataCloudObjectType.DMO)
|
|
276
|
-
return self._writer.write_to_dlo(name, dataframe, write_mode, **kwargs) # type: ignore[no-any-return]
|
|
277
|
-
|
|
278
|
-
def write_to_dmo(
|
|
279
|
-
self, name: str, dataframe: PySparkDataFrame, write_mode: WriteMode, **kwargs
|
|
280
|
-
) -> None:
|
|
281
|
-
"""Write a PySpark DataFrame to a DMO in Data Cloud.
|
|
282
|
-
|
|
283
|
-
Args:
|
|
284
|
-
name: The name of the DMO to write to.
|
|
285
|
-
dataframe: The PySpark DataFrame to write.
|
|
286
|
-
write_mode: The write mode to use for writing to the DMO.
|
|
287
|
-
"""
|
|
288
|
-
self._validate_data_layer_history_does_not_contain(DataCloudObjectType.DLO)
|
|
289
|
-
return self._writer.write_to_dmo(name, dataframe, write_mode, **kwargs) # type: ignore[no-any-return]
|
|
361
|
+
return cast(_ClientT, cls._instance)
|
|
290
362
|
|
|
291
363
|
def find_file_path(self, file_name: str) -> Path:
|
|
292
364
|
"""Resolve a bundled file shipped in the package to an absolute path.
|
|
@@ -358,6 +430,49 @@ class Client:
|
|
|
358
430
|
self._spark_llm_gateway = _build_spark_llm_gateway()
|
|
359
431
|
return self._spark_llm_gateway
|
|
360
432
|
|
|
433
|
+
def einstein_predict(
|
|
434
|
+
self,
|
|
435
|
+
model_api_name: str,
|
|
436
|
+
prediction_type: "PredictionType",
|
|
437
|
+
features: Dict[str, Any],
|
|
438
|
+
settings: Optional[Dict[str, Any]] = None,
|
|
439
|
+
) -> Dict[str, Any]:
|
|
440
|
+
"""Issue a one-shot Einstein prediction. This is the scalar counterpart
|
|
441
|
+
to :func:`einstein_predict_col`: it runs **once** — not per row. Use the
|
|
442
|
+
column helper method instead when you want to fan a prediction out
|
|
443
|
+
across every row of a DataFrame.
|
|
444
|
+
|
|
445
|
+
Example:
|
|
446
|
+
|
|
447
|
+
>>> from datacustomcode.einstein_predictions.types import PredictionType
|
|
448
|
+
>>> response = Client().einstein_predict(
|
|
449
|
+
... "my_regression_model",
|
|
450
|
+
... PredictionType.REGRESSION,
|
|
451
|
+
... {"square_feet": 1800, "beds": 3},
|
|
452
|
+
... )
|
|
453
|
+
|
|
454
|
+
Args:
|
|
455
|
+
model_api_name: API name of the Einstein model to invoke.
|
|
456
|
+
prediction_type: The :class:`PredictionType` of the model.
|
|
457
|
+
features: A mapping from model feature column name to a single
|
|
458
|
+
scalar value (``str`` / ``float`` / ``bool``).
|
|
459
|
+
settings: Optional prediction settings forwarded to the model.
|
|
460
|
+
|
|
461
|
+
Returns:
|
|
462
|
+
The prediction response payload as a plain Python ``dict``.
|
|
463
|
+
|
|
464
|
+
Raises:
|
|
465
|
+
EinsteinPredictionsCallError: If the prediction call fails.
|
|
466
|
+
"""
|
|
467
|
+
return self._get_spark_einstein_predictions().einstein_predict(
|
|
468
|
+
model_api_name, prediction_type, features, settings=settings
|
|
469
|
+
)
|
|
470
|
+
|
|
471
|
+
def _get_spark_einstein_predictions(self) -> SparkEinsteinPredictions:
|
|
472
|
+
if self._spark_einstein_predictions is None:
|
|
473
|
+
self._spark_einstein_predictions = _build_spark_einstein_predictions()
|
|
474
|
+
return self._spark_einstein_predictions
|
|
475
|
+
|
|
361
476
|
def _validate_data_layer_history_does_not_contain(
|
|
362
477
|
self, data_cloud_object_type: DataCloudObjectType
|
|
363
478
|
) -> None:
|
|
@@ -371,3 +486,115 @@ class Client:
|
|
|
371
486
|
|
|
372
487
|
def _record_dmo_access(self, name: str) -> None:
|
|
373
488
|
self._data_layer_history[DataCloudObjectType.DMO].add(name)
|
|
489
|
+
|
|
490
|
+
|
|
491
|
+
class Client(_BaseClient):
|
|
492
|
+
"""Entrypoint for batch access to Data Cloud objects.
|
|
493
|
+
|
|
494
|
+
This is the object used to read and write bounded snapshots of Data Cloud
|
|
495
|
+
DLOs and DMOs.
|
|
496
|
+
"""
|
|
497
|
+
|
|
498
|
+
_instance: ClassVar[Optional[Client]] = None
|
|
499
|
+
|
|
500
|
+
def read_dlo(self, name: str) -> PySparkDataFrame:
|
|
501
|
+
"""Read a DLO from Data Cloud.
|
|
502
|
+
|
|
503
|
+
Args:
|
|
504
|
+
name: The name of the DLO to read.
|
|
505
|
+
|
|
506
|
+
Returns:
|
|
507
|
+
A PySpark DataFrame containing the DLO data.
|
|
508
|
+
"""
|
|
509
|
+
self._record_dlo_access(name)
|
|
510
|
+
return self._reader.read_dlo(name) # type: ignore[no-any-return]
|
|
511
|
+
|
|
512
|
+
def read_dmo(self, name: str) -> PySparkDataFrame:
|
|
513
|
+
"""Read a DMO from Data Cloud.
|
|
514
|
+
|
|
515
|
+
Args:
|
|
516
|
+
name: The name of the DMO to read.
|
|
517
|
+
|
|
518
|
+
Returns:
|
|
519
|
+
A PySpark DataFrame containing the DMO data.
|
|
520
|
+
"""
|
|
521
|
+
self._record_dmo_access(name)
|
|
522
|
+
return self._reader.read_dmo(name) # type: ignore[no-any-return]
|
|
523
|
+
|
|
524
|
+
def write_to_dlo(
|
|
525
|
+
self, name: str, dataframe: PySparkDataFrame, write_mode: WriteMode, **kwargs
|
|
526
|
+
) -> None:
|
|
527
|
+
"""Write a PySpark DataFrame to a DLO in Data Cloud.
|
|
528
|
+
|
|
529
|
+
Args:
|
|
530
|
+
name: The name of the DLO to write to.
|
|
531
|
+
dataframe: The PySpark DataFrame to write.
|
|
532
|
+
write_mode: The write mode to use for writing to the DLO.
|
|
533
|
+
"""
|
|
534
|
+
self._validate_data_layer_history_does_not_contain(DataCloudObjectType.DMO)
|
|
535
|
+
return self._writer.write_to_dlo(name, dataframe, write_mode, **kwargs) # type: ignore[no-any-return]
|
|
536
|
+
|
|
537
|
+
def write_to_dmo(
|
|
538
|
+
self, name: str, dataframe: PySparkDataFrame, write_mode: WriteMode, **kwargs
|
|
539
|
+
) -> None:
|
|
540
|
+
"""Write a PySpark DataFrame to a DMO in Data Cloud.
|
|
541
|
+
|
|
542
|
+
Args:
|
|
543
|
+
name: The name of the DMO to write to.
|
|
544
|
+
dataframe: The PySpark DataFrame to write.
|
|
545
|
+
write_mode: The write mode to use for writing to the DMO.
|
|
546
|
+
"""
|
|
547
|
+
self._validate_data_layer_history_does_not_contain(DataCloudObjectType.DLO)
|
|
548
|
+
return self._writer.write_to_dmo(name, dataframe, write_mode, **kwargs) # type: ignore[no-any-return]
|
|
549
|
+
|
|
550
|
+
|
|
551
|
+
class StreamingClient(_BaseClient):
|
|
552
|
+
"""Entrypoint for streaming (``DELTA_SYNC``) access to Data Cloud objects.
|
|
553
|
+
|
|
554
|
+
This is the streaming counterpart to :class:`Client`. Instead of reading and
|
|
555
|
+
writing bounded snapshots, it reads a DLO/DMO change feed as a streaming
|
|
556
|
+
DataFrame and writes the transformed stream back via a ``StreamingQuery``.
|
|
557
|
+
"""
|
|
558
|
+
|
|
559
|
+
_instance: ClassVar[Optional[StreamingClient]] = None
|
|
560
|
+
|
|
561
|
+
def read_dlo_deltas(self) -> PySparkDataFrame:
|
|
562
|
+
"""Read the streaming change feed (deltas) for a DLO from Data Cloud.
|
|
563
|
+
|
|
564
|
+
For use in a streaming (``DELTA_SYNC``) BYOC transform. Returns a
|
|
565
|
+
streaming DataFrame whose rows carry the change-feed metadata columns
|
|
566
|
+
(``_record_type``, ``_commit_*``) alongside the source columns.
|
|
567
|
+
|
|
568
|
+
Returns:
|
|
569
|
+
A streaming PySpark DataFrame over the DLO change feed.
|
|
570
|
+
"""
|
|
571
|
+
self._record_dlo_access(_streaming_source_name())
|
|
572
|
+
return self._reader.read_dlo_deltas() # type: ignore[no-any-return]
|
|
573
|
+
|
|
574
|
+
def read_dmo_deltas(self) -> PySparkDataFrame:
|
|
575
|
+
"""Read the streaming change feed (deltas) for a DMO from Data Cloud.
|
|
576
|
+
|
|
577
|
+
Returns:
|
|
578
|
+
A streaming PySpark DataFrame over the DMO change feed.
|
|
579
|
+
"""
|
|
580
|
+
self._record_dmo_access(_streaming_source_name())
|
|
581
|
+
return self._reader.read_dmo_deltas() # type: ignore[no-any-return]
|
|
582
|
+
|
|
583
|
+
def write_dlo_deltas(
|
|
584
|
+
self, name: str, dataframe: PySparkDataFrame, **kwargs
|
|
585
|
+
) -> StreamingQuery:
|
|
586
|
+
"""Write a streaming DataFrame of deltas to a DLO in Data Cloud.
|
|
587
|
+
|
|
588
|
+
Starts a streaming query that writes each micro-batch to the
|
|
589
|
+
target DLO and returns the ``StreamingQuery`` handle; the caller
|
|
590
|
+
typically calls ``query.awaitTermination()``.
|
|
591
|
+
|
|
592
|
+
Args:
|
|
593
|
+
name: The name of the DLO to write to.
|
|
594
|
+
dataframe: The streaming PySpark DataFrame to write.
|
|
595
|
+
|
|
596
|
+
Returns:
|
|
597
|
+
The started ``StreamingQuery``.
|
|
598
|
+
"""
|
|
599
|
+
self._validate_data_layer_history_does_not_contain(DataCloudObjectType.DMO)
|
|
600
|
+
return self._writer.write_dlo_deltas(name, dataframe, **kwargs) # type: ignore[no-any-return]
|
datacustomcode/config.py
CHANGED
|
@@ -89,6 +89,9 @@ class ClientConfig(BaseConfig):
|
|
|
89
89
|
spark_provider_config: Union[
|
|
90
90
|
SparkProviderConfig[BaseSparkSessionProvider], None
|
|
91
91
|
] = None
|
|
92
|
+
# Source object name for a streaming (DELTA_SYNC) transform, populated by
|
|
93
|
+
# ``run_entrypoint`` from config.json's ``streamingSource`` field
|
|
94
|
+
streaming_source: Union[str, None] = None
|
|
92
95
|
|
|
93
96
|
def update(self, other: ClientConfig) -> ClientConfig:
|
|
94
97
|
"""Merge this ClientConfig with another, respecting force flags.
|
|
@@ -116,6 +119,8 @@ class ClientConfig(BaseConfig):
|
|
|
116
119
|
self.spark_provider_config = merge(
|
|
117
120
|
self.spark_provider_config, other.spark_provider_config
|
|
118
121
|
)
|
|
122
|
+
if other.streaming_source is not None:
|
|
123
|
+
self.streaming_source = other.streaming_source
|
|
119
124
|
return self
|
|
120
125
|
|
|
121
126
|
|
datacustomcode/config.yaml
CHANGED
|
@@ -14,9 +14,17 @@
|
|
|
14
14
|
# limitations under the License.
|
|
15
15
|
|
|
16
16
|
from datacustomcode.einstein_predictions.base import EinsteinPredictions
|
|
17
|
+
from datacustomcode.einstein_predictions.errors import EinsteinPredictionsCallError
|
|
17
18
|
from datacustomcode.einstein_predictions.impl.default import DefaultEinsteinPredictions
|
|
19
|
+
from datacustomcode.einstein_predictions.spark_base import SparkEinsteinPredictions
|
|
20
|
+
from datacustomcode.einstein_predictions.spark_default import (
|
|
21
|
+
DefaultSparkEinsteinPredictions,
|
|
22
|
+
)
|
|
18
23
|
|
|
19
24
|
__all__ = [
|
|
20
25
|
"DefaultEinsteinPredictions",
|
|
26
|
+
"DefaultSparkEinsteinPredictions",
|
|
21
27
|
"EinsteinPredictions",
|
|
28
|
+
"EinsteinPredictionsCallError",
|
|
29
|
+
"SparkEinsteinPredictions",
|
|
22
30
|
]
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# Copyright (c) 2025, Salesforce, Inc.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2
|
|
3
|
+
#
|
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
# you may not use this file except in compliance with the License.
|
|
6
|
+
# You may obtain a copy of the License at
|
|
7
|
+
#
|
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
#
|
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
# See the License for the specific language governing permissions and
|
|
14
|
+
# limitations under the License.
|
|
15
|
+
"""Exceptions raised by Einstein Predictions implementations."""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
from typing import Optional
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class EinsteinPredictionsCallError(RuntimeError):
|
|
23
|
+
"""Raised when an Einstein Predictions call returns an error."""
|
|
24
|
+
|
|
25
|
+
def __init__(
|
|
26
|
+
self,
|
|
27
|
+
message: str,
|
|
28
|
+
*,
|
|
29
|
+
status: Optional[object] = None,
|
|
30
|
+
error_code: Optional[str] = None,
|
|
31
|
+
error_message: Optional[str] = None,
|
|
32
|
+
) -> None:
|
|
33
|
+
super().__init__(message)
|
|
34
|
+
self.status = status
|
|
35
|
+
self.error_code = error_code
|
|
36
|
+
self.error_message = error_message
|