salesforce-data-customcode 6.0.7.dev1__py3-none-any.whl → 6.1.0.dev1__py3-none-any.whl

datacustomcode/einstein_predictions/spark_default.py

@@ -39,6 +39,9 @@ if TYPE_CHECKING:
 _STATUS_SUCCESS = "SUCCESS"
 _STATUS_ERROR = "ERROR"
 
+# HTTP status considered a successful prediction call.
+_HTTP_OK = 200
+
 
 class DefaultSparkEinsteinPredictions(SparkEinsteinPredictions):
 
@@ -104,6 +107,8 @@ class DefaultSparkEinsteinPredictions(SparkEinsteinPredictions):
 
         def _predict(values_row: Any) -> Dict[str, Optional[str]]:
             if values_row is None:
+                # An entirely null features struct is not the normal per-feature null
+                # case; surface it directly rather than masking it (debuggability).
                 return {
                     "status": _STATUS_ERROR,
                     "response": None,
@@ -181,6 +186,21 @@ def _call_predictions(
     return predictions.predict(request)
 
 
+def _null_feature_name(features: Dict[str, Any]) -> Optional[str]:
+    """Return the name of the first null feature value, or ``None``."""
+    for name, value in features.items():
+        if value is None:
+            return name
+    return None
+
+
+def _null_feature_message(name: str) -> str:
+    return (
+        f"Feature '{name}' has null value. Use coalesce() or when() to handle "
+        f"nulls before calling einstein_predict."
+    )
+
+
 def _invoke_predictions(
     predictions: "EinsteinPredictions",
     model_api_name: str,
@@ -190,18 +210,40 @@ def _invoke_predictions(
 ) -> Dict[str, Any]:
     from datacustomcode.einstein_predictions.errors import EinsteinPredictionsCallError
 
-    response = _call_predictions(
-        predictions, model_api_name, prediction_type, features, settings
-    )
-    if not response.is_success:
-        error_code = _extract_error_code(response)
+    null_feature = _null_feature_name(features)
+    if null_feature is not None:
+        message = _null_feature_message(null_feature)
+        raise EinsteinPredictionsCallError(
+            f"Einstein Predictions call failed: {message}",
+            status=None,
+            error_code=None,
+            error_message=message,
+        )
+
+    try:
+        response = _call_predictions(
+            predictions, model_api_name, prediction_type, features, settings
+        )
+    except EinsteinPredictionsCallError:
+        raise
+    except Exception as exc:
+        # Transport/build failures: surface the real error (no masking) so local
+        # runs stay debuggable. error_code stays None since there is no HTTP status.
+        raise EinsteinPredictionsCallError(
+            f"Einstein Predictions call failed: {exc}",
+            status=None,
+            error_code=None,
+            error_message=str(exc),
+        ) from exc
+
+    if response.status_code != _HTTP_OK:
+        error_message = json.dumps(response.data) if response.data is not None else None
         raise EinsteinPredictionsCallError(
             f"Einstein Predictions call failed: "
-            f"status_code={response.status_code}, "
-            f"error_code={error_code!r}, message={response.data!r}",
+            f"status_code={response.status_code}, message={error_message!r}",
             status=response.status_code,
-            error_code=error_code,
-            error_message=str(response.data) if response.data else None,
+            error_code=str(response.status_code),
+            error_message=error_message,
         )
     return response.data or {}
 
@@ -213,27 +255,46 @@ def _invoke_predictions_as_struct(
     features: Dict[str, Any],
     settings: Optional[Dict[str, Any]],
 ) -> Dict[str, Optional[str]]:
-    response = _call_predictions(
-        predictions, model_api_name, prediction_type, features, settings
-    )
-    if not response.is_success:
+    # (a) Customer-actionable data condition — surface the actionable message directly.
+    null_feature = _null_feature_name(features)
+    if null_feature is not None:
         return {
             "status": _STATUS_ERROR,
             "response": None,
-            "error_code": _extract_error_code(response),
-            "error_message": str(response.data) if response.data else None,
+            "error_code": None,
+            "error_message": _null_feature_message(null_feature),
         }
-    return {
-        "status": _STATUS_SUCCESS,
-        "response": json.dumps(response.data) if response.data is not None else None,
-        "error_code": None,
-        "error_message": None,
-    }
 
+    # (b) Transport/build failures — surface the real error (no masking) so local
+    # runs stay debuggable. error_code stays None since there is no HTTP status.
+    try:
+        response = _call_predictions(
+            predictions, model_api_name, prediction_type, features, settings
+        )
+    except Exception as exc:
+        return {
+            "status": _STATUS_ERROR,
+            "response": None,
+            "error_code": None,
+            "error_message": str(exc),
+        }
 
-def _extract_error_code(response: "PredictionResponse") -> Optional[str]:
-    if response.data:
-        error_code = response.data.get("errorCode")
-        if error_code is not None:
-            return str(error_code)
-    return None
+    if response.status_code == _HTTP_OK:
+        return {
+            "status": _STATUS_SUCCESS,
+            "response": (
+                json.dumps(response.data) if response.data is not None else None
+            ),
+            "error_code": None,
+            "error_message": None,
+        }
+
+    # (c) Non-200 SFAP HTTP error: error_code = status code, error_message = data JSON.
+    return {
+        "status": _STATUS_ERROR,
+        "response": None,
+        "error_code": str(response.status_code),
+        "error_message": (
+            json.dumps(response.data) if response.data is not None else None
+        ),
+    }

datacustomcode/io/reader/base.py

@@ -41,3 +41,45 @@ class BaseDataCloudReader(BaseDataAccessLayer):
         name: str,
         schema: Union[AtomicType, StructType, str, None] = None,
     ) -> PySparkDataFrame: ...
+
+    def read_dlo_deltas(self) -> PySparkDataFrame:
+        """Read the streaming change feed (deltas) for a Data Lake Object.
+
+        This is the streaming counterpart to :meth:`read_dlo`. It returns a
+        streaming DataFrame over the change feed the Data Cloud runtime
+        publishes for a streaming (``DELTA_SYNC``) transform. Concrete
+        streaming behavior is provided by the deployed Data Cloud runtime; the
+        base implementation raises :class:`NotImplementedError` so local
+        readers that do not support streaming fail clearly.
+
+        Returns:
+            A streaming PySpark DataFrame over the DLO change feed.
+
+        Raises:
+            NotImplementedError: If the active reader does not support streaming
+                deltas (e.g. the local development readers).
+        """
+        raise NotImplementedError(
+            "read_dlo_deltas is only supported when running in the Data Cloud "
+            "streaming runtime; the local reader does not support streaming "
+            "deltas."
+        )
+
+    def read_dmo_deltas(self) -> PySparkDataFrame:
+        """Read the streaming change feed (deltas) for a Data Model Object.
+
+        Streaming counterpart to :meth:`read_dmo`. See :meth:`read_dlo_deltas`
+        for behavior and the local-development caveat.
+
+        Returns:
+            A streaming PySpark DataFrame over the DMO change feed.
+
+        Raises:
+            NotImplementedError: If the active reader does not support streaming
+                deltas (e.g. the local development readers).
+        """
+        raise NotImplementedError(
+            "read_dmo_deltas is only supported when running in the Data Cloud "
+            "streaming runtime; the local reader does not support streaming "
+            "deltas."
+        )

datacustomcode/io/writer/base.py

@@ -22,6 +22,7 @@ from datacustomcode.io.base import BaseDataAccessLayer
 
 if TYPE_CHECKING:
     from pyspark.sql import DataFrame as PySparkDataFrame, SparkSession
+    from pyspark.sql.streaming import StreamingQuery
 
 
 class WriteMode(str, Enum):
@@ -57,3 +58,35 @@ class BaseDataCloudWriter(BaseDataAccessLayer):
     def write_to_dmo(
         self, name: str, dataframe: PySparkDataFrame, write_mode: WriteMode
     ) -> None: ...
+
+    def write_dlo_deltas(
+        self, name: str, dataframe: PySparkDataFrame
+    ) -> StreamingQuery:
+        """Write a streaming DataFrame of deltas to a Data Lake Object.
+
+        Streaming counterpart to :meth:`write_to_dlo`. Starts a streaming query
+        that writes each micro-batch to the target DLO via the Data Cloud
+        streaming sink and returns the resulting ``StreamingQuery`` handle. The
+        runtime owns the trigger and checkpoint location; callers pass only the
+        table name. Concrete streaming behavior is provided by the deployed
+        Data Cloud runtime; the base implementation raises
+        :class:`NotImplementedError`.
+
+        Args:
+            name: Target Data Lake Object name.
+            dataframe: Streaming PySpark DataFrame produced from a
+                ``read_dlo_deltas`` / ``read_dmo_deltas`` source.
+
+        Returns:
+            The started ``StreamingQuery``; the caller drives its lifecycle
+            (typically ``query.awaitTermination()``).
+
+        Raises:
+            NotImplementedError: If the active writer does not support streaming
+                deltas (e.g. the local development writers).
+        """
+        raise NotImplementedError(
+            "write_dlo_deltas is only supported when running in the Data Cloud "
+            "streaming runtime; the local writer does not support streaming "
+            "deltas."
+        )

datacustomcode/run.py

@@ -42,6 +42,15 @@ def _set_config_option(config_obj, key: str, value: Optional[str]) -> None:
         config_obj.options[key] = value
 
 
+def _read_streaming_source(config_json: dict) -> Optional[str]:
+    """Return the streaming source name from config.json's ``streamingSource``."""
+    source = config_json.get("streamingSource")
+    if not isinstance(source, dict):
+        return None
+    name = source.get("name")
+    return str(name) if name else None
+
+
 def _update_config_options(profile: Optional[str], sf_cli_org: Optional[str]):
     if sf_cli_org:
         config_key = "sf_cli_org"
@@ -125,6 +134,8 @@ def run_entrypoint(
         _set_config_option(config.reader_config, "dataspace", dataspace)
         _set_config_option(config.writer_config, "dataspace", dataspace)
 
+        config.streaming_source = _read_streaming_source(config_json)
+
     _update_config_options(profile, sf_cli_org)
 
     for dependency in dependencies:

datacustomcode/scan.py

@@ -15,6 +15,7 @@
 from __future__ import annotations
 
 import ast
+import copy
 import json
 import os
 import sys
@@ -32,6 +33,8 @@ import pydantic
 from datacustomcode.version import get_version
 
 DATA_ACCESS_METHODS = ["read_dlo", "read_dmo", "write_to_dlo", "write_to_dmo"]
+STREAMING_READ_METHODS = ["read_dlo_deltas", "read_dmo_deltas"]
+STREAMING_WRITE_METHODS = ["write_dlo_deltas"]
 
 DATA_TRANSFORM_CONFIG_TEMPLATE = {
     "sdkVersion": get_version(),
@@ -43,6 +46,20 @@ DATA_TRANSFORM_CONFIG_TEMPLATE = {
     },
 }
 
+STREAMING_TRANSFORM_CONFIG_TEMPLATE = {
+    "sdkVersion": get_version(),
+    "entryPoint": "",
+    "dataspace": "default",
+    "streamingSource": {
+        "type": "dlo",
+        "name": "",
+    },
+    "permissions": {
+        "read": {},
+        "write": {},
+    },
+}
+
 FUNCTION_CONFIG_TEMPLATE = {
     "entryPoint": "",
 }
@@ -160,6 +177,35 @@ class DataAccessLayerCalls(pydantic.BaseModel):
         return next(iter(self.write_to_dmo))
 
 
+class StreamingDataAccessLayerCalls(pydantic.BaseModel):
+    read_dlo_deltas: bool
+    read_dmo_deltas: bool
+    write_dlo_deltas: frozenset[str]
+
+    @pydantic.model_validator(mode="after")
+    def validate_access_layer(self) -> StreamingDataAccessLayerCalls:
+        if self.read_dlo_deltas and self.read_dmo_deltas:
+            raise ValueError(
+                "Cannot read DLO and DMO deltas in the same streaming transform."
+            )
+        if not self.read_dlo_deltas and not self.read_dmo_deltas:
+            raise ValueError(
+                "A streaming transform must read from at least one DLO or DMO "
+                "delta stream (read_dlo_deltas / read_dmo_deltas)."
+            )
+        if not self.write_dlo_deltas:
+            raise ValueError(
+                "A streaming transform must write to at least one DLO via "
+                "write_dlo_deltas."
+            )
+        return self
+
+    @property
+    def read_layer(self) -> str:
+        """Return the read source layer, ``"dlo"`` or ``"dmo"``."""
+        return "dlo" if self.read_dlo_deltas else "dmo"
+
+
 class ClientMethodVisitor(ast.NodeVisitor):
     """AST Visitor that finds all instances of Client read/write method calls."""
 
@@ -168,6 +214,9 @@ class ClientMethodVisitor(ast.NodeVisitor):
         self._read_dmo_instances: set[str] = set()
         self._write_to_dlo_instances: set[str] = set()
         self._write_to_dmo_instances: set[str] = set()
+        self._read_dlo_deltas: bool = False
+        self._read_dmo_deltas: bool = False
+        self._write_dlo_deltas_instances: set[str] = set()
         self.variable_values: Dict[str, Union[str, None]] = {}
 
     def visit_Assign(self, node: ast.Assign) -> None:
@@ -189,14 +238,15 @@ class ClientMethodVisitor(ast.NodeVisitor):
             node.func.value, ast.Name
         ):
             method_name = node.func.attr
+
+            if method_name == "read_dlo_deltas":
+                self._read_dlo_deltas = True
+            elif method_name == "read_dmo_deltas":
+                self._read_dmo_deltas = True
+
             if method_name in DATA_ACCESS_METHODS and node.args:
                 arg = node.args[0]
-                name = None
-
-                if isinstance(arg, ast.Constant) and isinstance(arg.value, str):
-                    name = arg.value
-                elif isinstance(arg, ast.Name) and arg.id in self.variable_values:
-                    name = self.variable_values[arg.id]
+                name = self._resolve_name_arg(arg)
 
                 if name:
                     if method_name == "read_dlo":
@@ -207,8 +257,29 @@ class ClientMethodVisitor(ast.NodeVisitor):
                         self._write_to_dlo_instances.add(name)
                     elif method_name == "write_to_dmo":
                         self._write_to_dmo_instances.add(name)
+            elif method_name in STREAMING_WRITE_METHODS and node.args:
+                name = self._resolve_name_arg(node.args[0])
+                if name and method_name == "write_dlo_deltas":
+                    self._write_dlo_deltas_instances.add(name)
         self.generic_visit(node)
 
+    def _resolve_name_arg(self, arg: ast.expr) -> Union[str, None]:
+        """Resolve a string-literal or tracked-variable first argument."""
+        if isinstance(arg, ast.Constant) and isinstance(arg.value, str):
+            return arg.value
+        if isinstance(arg, ast.Name) and arg.id in self.variable_values:
+            return self.variable_values[arg.id]
+        return None
+
+    @property
+    def is_streaming(self) -> bool:
+        """Whether any streaming (delta) access method was found."""
+        return (
+            self._read_dlo_deltas
+            or self._read_dmo_deltas
+            or bool(self._write_dlo_deltas_instances)
+        )
+
     def found(self) -> DataAccessLayerCalls:
         return DataAccessLayerCalls(
             read_dlo=frozenset(self._read_dlo_instances),
@@ -217,6 +288,13 @@ class ClientMethodVisitor(ast.NodeVisitor):
             write_to_dmo=frozenset(self._write_to_dmo_instances),
         )
 
+    def found_streaming(self) -> StreamingDataAccessLayerCalls:
+        return StreamingDataAccessLayerCalls(
+            read_dlo_deltas=self._read_dlo_deltas,
+            read_dmo_deltas=self._read_dmo_deltas,
+            write_dlo_deltas=frozenset(self._write_dlo_deltas_instances),
+        )
+
 
 class ImportVisitor(ast.NodeVisitor):
     """AST Visitor that extracts external package imports from Python code."""
@@ -301,23 +379,51 @@ def write_requirements_file(file_path: str) -> str:
     return requirements_path
 
 
-def scan_file(file_path: str) -> DataAccessLayerCalls:
-    """Scan a single Python file for Client read/write method calls."""
+def _visit_file(file_path: str) -> ClientMethodVisitor:
+    """Parse a Python file and return the populated method visitor."""
     with open(file_path, "r") as f:
-        code = f.read()
-        tree = ast.parse(code)
-        visitor = ClientMethodVisitor()
-        visitor.visit(tree)
-        return visitor.found()
+        tree = ast.parse(f.read())
+    visitor = ClientMethodVisitor()
+    visitor.visit(tree)
+    return visitor
+
+
+def scan_file(file_path: str) -> DataAccessLayerCalls:
+    """Scan a single Python file for batch Client read/write method calls."""
+    return _visit_file(file_path).found()
+
+
+def scan_file_streaming(file_path: str) -> StreamingDataAccessLayerCalls:
+    """Scan a single Python file for StreamingClient delta method calls."""
+    return _visit_file(file_path).found_streaming()
+
 
+def file_is_streaming(file_path: str) -> bool:
+    """Return whether the entrypoint uses streaming (delta) access methods."""
+    return _visit_file(file_path).is_streaming
 
-def dc_config_json_from_file(file_path: str, type: str) -> dict[str, Any]:
-    """Create a Data Cloud Custom Code config JSON from a script."""
+
+def dc_config_json_from_file(
+    file_path: str, type: str, streaming: bool = False
+) -> dict[str, Any]:
+    """Create a Data Cloud Custom Code config JSON from a script.
+
+    Args:
+        file_path: Path to the entrypoint.
+        type: Package type, ``"script"`` or ``"function"``.
+        streaming: For scripts, a streaming
+            (``streamingSource``) config instead of a batch one.
+    """
     config: dict[str, Any]
     if type == "script":
-        config = DATA_TRANSFORM_CONFIG_TEMPLATE.copy()
+        template = (
+            STREAMING_TRANSFORM_CONFIG_TEMPLATE
+            if streaming
+            else DATA_TRANSFORM_CONFIG_TEMPLATE
+        )
+        config = copy.deepcopy(template)
     elif type == "function":
-        config = FUNCTION_CONFIG_TEMPLATE.copy()
+        config = copy.deepcopy(FUNCTION_CONFIG_TEMPLATE)
     config["entryPoint"] = os.path.basename(file_path)
     return config
 
@@ -372,22 +478,51 @@ def update_config(file_path: str) -> dict[str, Any]:
 
     if package_type == "script":
         existing_config["dataspace"] = get_dataspace(existing_config)
-        output = scan_file(file_path)
-        read: dict[str, list[str]] = {}
-        if output.read_dlo:
-            read["dlo"] = list(output.read_dlo)
-        else:
-            read["dmo"] = list(output.read_dmo)
-        write: dict[str, list[str]] = {}
-        if output.write_to_dlo:
-            write["dlo"] = list(output.write_to_dlo)
+        if file_is_streaming(file_path):
+            _update_streaming_config(existing_config, file_path)
         else:
-            write["dmo"] = list(output.write_to_dmo)
-
-        existing_config["permissions"] = {"read": read, "write": write}
+            existing_config.pop("streamingSource", None)
+            output = scan_file(file_path)
+            read: dict[str, list[str]] = {}
+            if output.read_dlo:
+                read["dlo"] = list(output.read_dlo)
+            else:
+                read["dmo"] = list(output.read_dmo)
+            write: dict[str, list[str]] = {}
+            if output.write_to_dlo:
+                write["dlo"] = list(output.write_to_dlo)
+            else:
+                write["dmo"] = list(output.write_to_dmo)
+
+            existing_config["permissions"] = {"read": read, "write": write}
     return existing_config
 
 
+def _update_streaming_config(existing_config: dict[str, Any], file_path: str) -> None:
+    output = scan_file_streaming(file_path)
+    read_layer = output.read_layer
+
+    source = existing_config.get("streamingSource")
+    if not isinstance(source, dict):
+        source = {}
+    source_name = source.get("name", "")
+    existing_config["streamingSource"] = {"type": read_layer, "name": source_name}
+
+    if not source_name:
+        logger.warning(
+            "streamingSource.name is empty in config.json. A streaming "
+            "transform must declare its read source; set streamingSource.name "
+            "to the DLO/DMO the transform reads from."
+        )
+
+    read_names = [source_name] if source_name else []
+    write_names = list(output.write_dlo_deltas)
+    existing_config["permissions"] = {
+        "read": {read_layer: read_names},
+        "write": {"dlo": write_names},
+    }
+
+
 def get_dataspace(existing_config: dict[str, str]) -> str:
     if "dataspace" in existing_config:
         dataspace_value = existing_config["dataspace"]

datacustomcode/template.py

@@ -23,8 +23,12 @@ from datacustomcode.constants import FEATURE_TEMPLATE_MAPPING
 script_template_dir = os.path.join(os.path.dirname(__file__), "templates", "script")
 function_template_dir = os.path.join(os.path.dirname(__file__), "templates", "function")
 
+STREAMING_EXAMPLE_ENTRYPOINT = os.path.join(
+    script_template_dir, "examples", "streaming_deltas", "entrypoint.py"
+)
 
-def copy_script_template(target_dir: str) -> None:
+
+def copy_script_template(target_dir: str, streaming: bool = False) -> None:
     """Copy the template to the target directory."""
     os.makedirs(target_dir, exist_ok=True)
 
@@ -39,6 +43,14 @@ def copy_script_template(target_dir: str) -> None:
             logger.debug(f"Copying file {source} to {destination}...")
             shutil.copy2(source, destination)
 
+    if streaming:
+        destination = os.path.join(target_dir, "payload", "entrypoint.py")
+        logger.debug(
+            f"Copying streaming example {STREAMING_EXAMPLE_ENTRYPOINT} to "
+            f"{destination}..."
+        )
+        shutil.copy2(STREAMING_EXAMPLE_ENTRYPOINT, destination)
+
 
 def copy_function_template(target_dir: str, use_in_feature: Optional[str]) -> None:
     os.makedirs(target_dir, exist_ok=True)

datacustomcode/templates/script/examples/streaming_deltas/entrypoint.py

@@ -0,0 +1,49 @@
+"""Streaming BYOC transform: read a DLO change feed and write the deltas back.
+
+This example is the streaming counterpart to a normal batch entrypoint. Instead
+of a batch ``Client`` with ``read_dlo`` / ``write_to_dlo`` (which read and write
+a bounded snapshot), it uses a :class:`StreamingClient` and its streaming delta
+methods:
+
+* ``client.read_dlo_deltas()`` returns a *streaming* DataFrame over the
+  Change Data Feed of the source DLO. Each row carries the source columns plus
+  change-feed metadata columns (``_record_type``, ``_commit_*``).
+* ``client.write_dlo_deltas(name, df)`` starts a streaming query that writes
+  each micro-batch to the target DLO and returns the ``StreamingQuery`` handle.
+  The runtime owns the trigger, and checkpoint location — the caller only
+  chooses the table.
+
+The transform in between is ordinary PySpark. Because the source is a change
+feed, keep the metadata columns on the DataFrame you hand to
+``write_dlo_deltas`` — the sink relies on them to merge changes correctly.
+
+This entrypoint only runs inside the Data Cloud streaming (``DELTA_SYNC``)
+runtime; the local ``datacustomcode run`` readers/writers raise
+``NotImplementedError`` for the delta methods.
+"""
+
+from pyspark.sql.functions import col, upper
+
+from datacustomcode.client import StreamingClient
+
+
+def main():
+    client = StreamingClient()
+
+    # Streaming DataFrame over the source DLO's change feed.
+    deltas = client.read_dlo_deltas()
+
+    # Ordinary PySpark transform.
+    transformed = deltas.withColumn("description__c", upper(col("description__c")))
+
+    # Start the streaming write. write_dlo_deltas returns the StreamingQuery;
+    # the trigger and checkpoint location are provided by the runtime.
+    query = client.write_dlo_deltas("Account_std_copy__dll", transformed)
+
+    # Drive the query's lifecycle. In the streaming runtime this blocks until
+    # the job is stopped by the platform.
+    query.awaitTermination()
+
+
+if __name__ == "__main__":
+    main()