salesforce-data-customcode 6.1.0.dev3__py3-none-any.whl → 6.1.0.dev5__py3-none-any.whl

datacustomcode/config.yaml

@@ -34,9 +34,3 @@ llm_gateway_config:
 
 spark_llm_gateway_config:
   type_config_name: DefaultSparkLLMGateway
-
-named_credential_config:
-  type_config_name: DefaultNamedCredential
-
-spark_named_credential_config:
-  type_config_name: DefaultSparkNamedCredential

datacustomcode/constants.py

@@ -35,9 +35,17 @@ FEATURE_TEMPLATE_MAPPING = {
 
 # Feature name to Connect API name mapping
 USE_IN_FEATURE_MAPPING_FOR_CONNECT_API = {
-    "SearchIndexChunking": "SearchIndexChunking",
+    "SearchIndexChunking": "UnstructuredChunking",
 }
 
+# Script (data transform) invoke options
+SCRIPT_USE_IN_FEATURE_BATCH = "BatchTransform"
+SCRIPT_USE_IN_FEATURE_STREAMING = "StreamingTransform"
+SCRIPT_USE_IN_FEATURE_OPTIONS = [
+    SCRIPT_USE_IN_FEATURE_BATCH,
+    SCRIPT_USE_IN_FEATURE_STREAMING,
+]
+
 # Pydantic request/response type names to feature names
 REQUEST_TYPE_TO_FEATURE = {
     "SearchIndexChunkingV1Request": "SearchIndexChunking",

datacustomcode/deploy.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 from __future__ import annotations
 
+import copy
 from html import unescape
 import json
 import os
@@ -37,13 +38,11 @@ import requests
 
 from datacustomcode.cmd import cmd_output
 from datacustomcode.constants import REQUEST_TYPE_TO_FEATURE
-from datacustomcode.named_credential.direct.credentials import (
-    EXTERNAL_CALLOUT_CREDENTIAL,
-)
 from datacustomcode.scan import find_base_directory, get_package_type
 
-DATA_CUSTOM_CODE_PATH = "services/data/v67.0/ssot/data-custom-code"
+DATA_CUSTOM_CODE_PATH = "services/data/v63.0/ssot/data-custom-code"
 DATA_TRANSFORMS_PATH = "services/data/v63.0/ssot/data-transforms"
+DATA_CUSTOM_CODE_INVOKE_OPTIONS_PATH = "services/data/v67.0/ssot/data-custom-code"
 WAIT_FOR_DEPLOYMENT_TIMEOUT = 3000
 
 # Available compute types for Data Cloud deployments.
@@ -110,6 +109,7 @@ class CodeExtensionMetadata(BaseModel):
     description: str
     computeType: str
     codeType: str
+    functionInvokeOptions: Union[list[str], None] = None
     invokeOptions: Union[list[str], None] = None
 
     def __init__(self, **data):
@@ -203,7 +203,14 @@ def create_deployment(
     access_token: AccessTokenResponse, metadata: CodeExtensionMetadata
 ) -> CreateDeploymentResponse:
     """Create a custom code deployment in the DataCloud."""
-    url = _join_strip_url(access_token.instance_url, DATA_CUSTOM_CODE_PATH)
+    # invokeOptions only binds at v67.0; route there when it is set so the
+    # option isn't silently dropped. Everything else stays on v63.0.
+    code_custom_code_path = (
+        DATA_CUSTOM_CODE_INVOKE_OPTIONS_PATH
+        if metadata.invokeOptions
+        else DATA_CUSTOM_CODE_PATH
+    )
+    url = _join_strip_url(access_token.instance_url, code_custom_code_path)
     body = dict[str, Any](
         {
             "label": metadata.name,
@@ -214,6 +221,8 @@ def create_deployment(
             "codeType": metadata.codeType,
         }
     )
+    if metadata.functionInvokeOptions:
+        body["functionInvokeOptions"] = metadata.functionInvokeOptions
     if metadata.invokeOptions:
         body["invokeOptions"] = metadata.invokeOptions
     logger.debug(f"Creating deployment {metadata.name}...")
@@ -239,8 +248,6 @@ DEPENDENCIES_ARCHIVE_PATH = os.path.join(
 )
 PY_FILES_PATH = os.path.join("payload", "py-files")
 ZIP_FILE_NAME = "deployment.zip"
-# Local-only files that must never be packaged into the deployment zip.
-EXCLUDED_FILES = (".DS_Store", EXTERNAL_CALLOUT_CREDENTIAL)
 
 
 def prepare_dependency_archive(
@@ -393,6 +400,30 @@ class DataTransformConfig(BaseConfig):
     dataspace: str
     permissions: Permissions
     dataObjects: Optional[list[DataObject]] = None
+    streamingSource: Optional[StreamingSource] = None
+
+    @property
+    def is_streaming(self) -> bool:
+        return self.streamingSource is not None
+
+    @model_validator(mode="after")
+    def _validate_layers(self) -> "DataTransformConfig":
+        read_is_dlo = isinstance(self.permissions.read, DloPermission)
+        write_is_dlo = isinstance(self.permissions.write, DloPermission)
+        if self.is_streaming:
+            if not write_is_dlo:
+                raise ValueError(
+                    "A streaming transform must write to a DLO "
+                    "(permissions.write must be a 'dlo' entry)."
+                )
+        elif read_is_dlo != write_is_dlo:
+            raise ValueError(
+                "permissions.read and permissions.write must both reference "
+                "DLOs or both reference DMOs (got "
+                f"read={type(self.permissions.read).__name__}, "
+                f"write={type(self.permissions.write).__name__})"
+            )
+        return self
 
 
 class FunctionConfig(BaseConfig):
@@ -407,23 +438,15 @@ class DmoPermission(BaseModel):
     dmo: list[str]
 
 
+class StreamingSource(BaseModel):
+    type: str
+    name: str
+
+
 class Permissions(BaseModel):
     read: Union[DloPermission, DmoPermission]
     write: Union[DloPermission, DmoPermission]
 
-    @model_validator(mode="after")
-    def _no_mixed_layers(self) -> "Permissions":
-        read_is_dlo = isinstance(self.read, DloPermission)
-        write_is_dlo = isinstance(self.write, DloPermission)
-        if read_is_dlo != write_is_dlo:
-            raise ValueError(
-                "permissions.read and permissions.write must both reference "
-                "DLOs or both reference DMOs (got "
-                f"read={type(self.read).__name__}, "
-                f"write={type(self.write).__name__})"
-            )
-        return self
-
 
 def _permission_entries(perm: Union[DloPermission, DmoPermission]) -> list[str]:
     """Return the list of object names regardless of layer (DLO or DMO)."""
@@ -495,7 +518,9 @@ def create_data_transform(
 ) -> dict:
     """Create a data transform in the DataCloud."""
     script_name = metadata.name
-    request_hydrated = DATA_TRANSFORM_REQUEST_TEMPLATE.copy()
+    # Deep copy: the template's nested nodes/sources/macros dicts would
+    # otherwise be shared across calls and accumulate entries between deploys.
+    request_hydrated = copy.deepcopy(DATA_TRANSFORM_REQUEST_TEMPLATE)
 
     # Add nodes for each write entry (DLO or DMO)
     for i, name in enumerate(
@@ -538,7 +563,7 @@ def create_data_transform(
         "definition": definition,
         "label": f"{metadata.name}",
         "name": f"{metadata.name}",
-        "type": "BATCH",
+        "type": "STREAMING" if data_transform_config.is_streaming else "BATCH",
         "dataSpaceName": data_transform_config.dataspace,
     }
 
@@ -603,9 +628,9 @@ def zip(
 
     with zipfile.ZipFile(ZIP_FILE_NAME, "w", zipfile.ZIP_DEFLATED) as zipf:
         for root, dirs, files in os.walk(directory):
-            # Skip .DS_Store and local credentials.
+            # Skip .DS_Store files when adding to zip
             for file in files:
-                if file not in EXCLUDED_FILES:
+                if file != ".DS_Store":
                     abs_path = os.path.join(root, file)
                     arcname = os.path.relpath(abs_path, directory)
                     zipf.write(abs_path, arcname)
@@ -621,9 +646,18 @@ def deploy_full(
     callback=None,
 ) -> AccessTokenResponse:
     """Deploy a data transform in the DataCloud."""
+    from datacustomcode.constants import SCRIPT_USE_IN_FEATURE_STREAMING
+
     # prepare payload
     config = get_config(directory)
 
+    if (
+        isinstance(config, DataTransformConfig)
+        and config.is_streaming
+        and not metadata.invokeOptions
+    ):
+        metadata.invokeOptions = [SCRIPT_USE_IN_FEATURE_STREAMING]
+
     # create deployment and upload payload
     deployment = create_deployment(access_token, metadata)
     zip(directory, docker_network, metadata.codeType)

datacustomcode/function/runtime.py

@@ -23,8 +23,6 @@ from datacustomcode.file.path.default import DefaultFindFilePath
 from datacustomcode.function.base import BaseRuntime
 from datacustomcode.llm_gateway.base import LLMGateway
 from datacustomcode.llm_gateway_config import llm_gateway_config
-from datacustomcode.named_credential.base import NamedCredential
-from datacustomcode.named_credential_config import named_credential_config
 
 
 class Runtime(BaseRuntime):
@@ -71,7 +69,6 @@ class Runtime(BaseRuntime):
         self._llm_gateway: Optional[LLMGateway] = None
         self._file = DefaultFindFilePath()
         self._einstein_predictions: Optional[EinsteinPredictions] = None
-        self._named_credential: Optional[NamedCredential] = None
 
     @property
     def llm_gateway(self) -> LLMGateway:
@@ -101,16 +98,3 @@ class Runtime(BaseRuntime):
                 einstein_predictions_config.einstein_predictions_config.to_object()
             )
         return self._einstein_predictions
-
-    @property
-    def named_credential(self) -> NamedCredential:
-        if self._named_credential is None:
-            if named_credential_config.named_credential_config is None:
-                raise RuntimeError(
-                    "Named Credential is not configured. Add "
-                    "'named_credential_config' section to config.yaml"
-                )
-            self._named_credential = (
-                named_credential_config.named_credential_config.to_object()
-            )
-        return self._named_credential

datacustomcode/io/reader/base.py

@@ -41,3 +41,45 @@ class BaseDataCloudReader(BaseDataAccessLayer):
         name: str,
         schema: Union[AtomicType, StructType, str, None] = None,
     ) -> PySparkDataFrame: ...
+
+    def read_dlo_deltas(self) -> PySparkDataFrame:
+        """Read the streaming change feed (deltas) for a Data Lake Object.
+
+        This is the streaming counterpart to :meth:`read_dlo`. It returns a
+        streaming DataFrame over the change feed the Data Cloud runtime
+        publishes for a streaming (``DELTA_SYNC``) transform. Concrete
+        streaming behavior is provided by the deployed Data Cloud runtime; the
+        base implementation raises :class:`NotImplementedError` so local
+        readers that do not support streaming fail clearly.
+
+        Returns:
+            A streaming PySpark DataFrame over the DLO change feed.
+
+        Raises:
+            NotImplementedError: If the active reader does not support streaming
+                deltas (e.g. the local development readers).
+        """
+        raise NotImplementedError(
+            "read_dlo_deltas is only supported when running in the Data Cloud "
+            "streaming runtime; the local reader does not support streaming "
+            "deltas."
+        )
+
+    def read_dmo_deltas(self) -> PySparkDataFrame:
+        """Read the streaming change feed (deltas) for a Data Model Object.
+
+        Streaming counterpart to :meth:`read_dmo`. See :meth:`read_dlo_deltas`
+        for behavior and the local-development caveat.
+
+        Returns:
+            A streaming PySpark DataFrame over the DMO change feed.
+
+        Raises:
+            NotImplementedError: If the active reader does not support streaming
+                deltas (e.g. the local development readers).
+        """
+        raise NotImplementedError(
+            "read_dmo_deltas is only supported when running in the Data Cloud "
+            "streaming runtime; the local reader does not support streaming "
+            "deltas."
+        )

datacustomcode/io/writer/base.py

@@ -22,6 +22,7 @@ from datacustomcode.io.base import BaseDataAccessLayer
 
 if TYPE_CHECKING:
     from pyspark.sql import DataFrame as PySparkDataFrame, SparkSession
+    from pyspark.sql.streaming import StreamingQuery
 
 
 class WriteMode(str, Enum):
@@ -57,3 +58,47 @@ class BaseDataCloudWriter(BaseDataAccessLayer):
     def write_to_dmo(
         self, name: str, dataframe: PySparkDataFrame, write_mode: WriteMode
     ) -> None: ...
+
+    def auto_write_to_dlo(self, name: str, dataframe: PySparkDataFrame) -> None:
+        """Write to a DLO automatically picking the write mode.
+        For use with streaming transforms when running in rebuild or initial sync mode.
+        """
+        raise NotImplementedError
+
+    def auto_write_to_dmo(self, name: str, dataframe: PySparkDataFrame) -> None:
+        """Write to a DMO automatically picking the write mode.
+        For use with streaming transforms when running in rebuild or initial sync mode.
+        """
+        raise NotImplementedError
+
+    def write_dlo_deltas(
+        self, name: str, dataframe: PySparkDataFrame
+    ) -> StreamingQuery:
+        """Write a streaming DataFrame of deltas to a Data Lake Object.
+
+        Streaming counterpart to :meth:`write_to_dlo`. Starts a streaming query
+        that writes each micro-batch to the target DLO via the Data Cloud
+        streaming sink and returns the resulting ``StreamingQuery`` handle. The
+        runtime owns the trigger and checkpoint location; callers pass only the
+        table name. Concrete streaming behavior is provided by the deployed
+        Data Cloud runtime; the base implementation raises
+        :class:`NotImplementedError`.
+
+        Args:
+            name: Target Data Lake Object name.
+            dataframe: Streaming PySpark DataFrame produced from a
+                ``read_dlo_deltas`` / ``read_dmo_deltas`` source.
+
+        Returns:
+            The started ``StreamingQuery``; the caller drives its lifecycle
+            (typically ``query.awaitTermination()``).
+
+        Raises:
+            NotImplementedError: If the active writer does not support streaming
+                deltas (e.g. the local development writers).
+        """
+        raise NotImplementedError(
+            "write_dlo_deltas is only supported when running in the Data Cloud "
+            "streaming runtime; the local writer does not support streaming "
+            "deltas."
+        )

datacustomcode/io/writer/csv.py

@@ -36,6 +36,10 @@ class CSVDataCloudWriter(BaseDataCloudWriter):
             name = f"{name}{SUFFIX}"
         dataframe.write.csv(name, mode=write_mode)
 
+    def auto_write_to_dlo(self, name: str, dataframe: PySparkDataFrame) -> None:
+        # use overwrite since this is a local only writer
+        self.write_to_dlo(name, dataframe, WriteMode.OVERWRITE)
+
     def write_to_dmo(
         self, name: str, dataframe: PySparkDataFrame, write_mode: WriteMode
     ) -> None:
@@ -43,3 +47,7 @@ class CSVDataCloudWriter(BaseDataCloudWriter):
         if not name.lower().endswith(SUFFIX):
             name = f"{name}{SUFFIX}"
         dataframe.write.csv(name, mode=write_mode)
+
+    def auto_write_to_dmo(self, name: str, dataframe: PySparkDataFrame) -> None:
+        # use overwrite since this is a local only writer
+        self.write_to_dmo(name, dataframe, WriteMode.OVERWRITE)

datacustomcode/io/writer/print.py

@@ -122,6 +122,10 @@ class PrintDataCloudWriter(BaseDataCloudWriter):
 
         dataframe.show()
 
+    def auto_write_to_dlo(self, name: str, dataframe: PySparkDataFrame) -> None:
+        self.validate_dataframe_columns_against_dlo(dataframe, name)
+        dataframe.show()
+
     def write_to_dmo(
         self, name: str, dataframe: PySparkDataFrame, write_mode: WriteMode
     ) -> None:
@@ -130,3 +134,6 @@ class PrintDataCloudWriter(BaseDataCloudWriter):
         # so just show the dataframe.
 
         dataframe.show()
+
+    def auto_write_to_dmo(self, name: str, dataframe: PySparkDataFrame) -> None:
+        dataframe.show()

datacustomcode/run.py

@@ -27,7 +27,6 @@ from typing import (
 from datacustomcode.config import config
 from datacustomcode.einstein_predictions_config import einstein_predictions_config
 from datacustomcode.llm_gateway_config import llm_gateway_config
-from datacustomcode.named_credential_config import named_credential_config
 from datacustomcode.scan import find_base_directory, get_package_type
 
 
@@ -43,6 +42,15 @@ def _set_config_option(config_obj, key: str, value: Optional[str]) -> None:
         config_obj.options[key] = value
 
 
+def _read_streaming_source(config_json: dict) -> Optional[str]:
+    """Return the streaming source name from config.json's ``streamingSource``."""
+    source = config_json.get("streamingSource")
+    if not isinstance(source, dict):
+        return None
+    name = source.get("name")
+    return str(name) if name else None
+
+
 def _update_config_options(profile: Optional[str], sf_cli_org: Optional[str]):
     if sf_cli_org:
         config_key = "sf_cli_org"
@@ -56,9 +64,6 @@ def _update_config_options(profile: Optional[str], sf_cli_org: Optional[str]):
         _set_config_option(
             llm_gateway_config.llm_gateway_config, config_key, sf_cli_org
         )
-        _set_config_option(
-            named_credential_config.named_credential_config, config_key, sf_cli_org
-        )
     elif profile != "default":
         config_key = "credentials_profile"
         _set_config_option(config.reader_config, config_key, profile)
@@ -67,9 +72,6 @@ def _update_config_options(profile: Optional[str], sf_cli_org: Optional[str]):
             einstein_predictions_config.einstein_predictions_config, config_key, profile
         )
         _set_config_option(llm_gateway_config.llm_gateway_config, config_key, profile)
-        _set_config_option(
-            named_credential_config.named_credential_config, config_key, profile
-        )
 
 
 def run_entrypoint(
@@ -132,6 +134,8 @@ def run_entrypoint(
         _set_config_option(config.reader_config, "dataspace", dataspace)
         _set_config_option(config.writer_config, "dataspace", dataspace)
 
+        config.streaming_source = _read_streaming_source(config_json)
+
     _update_config_options(profile, sf_cli_org)
 
     for dependency in dependencies: