salesforce-data-customcode 6.0.6.dev2__tar.gz → 6.0.6.dev3__tar.gz

{salesforce_data_customcode-6.0.6.dev2 → salesforce_data_customcode-6.0.6.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: salesforce-data-customcode
-Version: 6.0.6.dev2
+Version: 6.0.6.dev3
 Summary: Data Cloud Custom Code SDK
 License-Expression: Apache-2.0
 License-File: LICENSE.txt

{salesforce_data_customcode-6.0.6.dev2 → salesforce_data_customcode-6.0.6.dev3}/pyproject.toml

@@ -18,7 +18,7 @@ license = "Apache-2.0"
 name = "salesforce-data-customcode"
 readme = "README.md"
 requires-python = ">=3.10,<3.12"
-version = "6.0.6.dev2"
+version = "6.0.6.dev3"
 
 [tool.black]
 exclude = '''

{salesforce_data_customcode-6.0.6.dev2 → salesforce_data_customcode-6.0.6.dev3}/src/datacustomcode/cli.py

@@ -283,10 +283,19 @@ def deploy(
 )
 @click.option(
     "--use-in-feature",
-    default="SearchIndexChunking",
-    help="Feature where this function will be used (only applicable for function).",
+    "-u",
+    default=None,
+    help=(
+        "Invoke option for this package. For scripts: 'BatchTransform' "
+        "(default) or 'StreamingTransform'. For functions: 'SearchIndexChunking'."
+    ),
 )
 def init(directory: str, code_type: str, use_in_feature: Optional[str]):
+    from datacustomcode.constants import (
+        SCRIPT_USE_IN_FEATURE_BATCH,
+        SCRIPT_USE_IN_FEATURE_OPTIONS,
+        SCRIPT_USE_IN_FEATURE_STREAMING,
+    )
     from datacustomcode.scan import (
         dc_config_json_from_file,
         update_config,
@@ -294,9 +303,23 @@ def init(directory: str, code_type: str, use_in_feature: Optional[str]):
     )
     from datacustomcode.template import copy_function_template, copy_script_template
 
+    streaming = False
+    if code_type == "script":
+        use_in_feature = use_in_feature or SCRIPT_USE_IN_FEATURE_BATCH
+        if use_in_feature not in SCRIPT_USE_IN_FEATURE_OPTIONS:
+            click.secho(
+                f"Error: Invalid --use-in-feature '{use_in_feature}' for a "
+                f"script. Valid options: {', '.join(SCRIPT_USE_IN_FEATURE_OPTIONS)}.",
+                fg="red",
+            )
+            raise click.Abort()
+        streaming = use_in_feature == SCRIPT_USE_IN_FEATURE_STREAMING
+    else:
+        use_in_feature = use_in_feature or "SearchIndexChunking"
+
     click.echo("Copying template to " + click.style(directory, fg="blue", bold=True))
     if code_type == "script":
-        copy_script_template(directory)
+        copy_script_template(directory, streaming=streaming)
     elif code_type == "function":
         copy_function_template(directory, use_in_feature)
     entrypoint_path = os.path.join(directory, PAYLOAD_DIR, ENTRYPOINT_FILE)
@@ -306,7 +329,9 @@ def init(directory: str, code_type: str, use_in_feature: Optional[str]):
     sdk_config = {"type": code_type}
     write_sdk_config(directory, sdk_config)
 
-    config_json = dc_config_json_from_file(entrypoint_path, code_type)
+    config_json = dc_config_json_from_file(
+        entrypoint_path, code_type, streaming=streaming
+    )
     with open(config_location, "w") as f:
         json.dump(config_json, f, indent=2)
 

{salesforce_data_customcode-6.0.6.dev2 → salesforce_data_customcode-6.0.6.dev3}/src/datacustomcode/constants.py

@@ -38,6 +38,14 @@ USE_IN_FEATURE_MAPPING_FOR_CONNECT_API = {
     "SearchIndexChunking": "UnstructuredChunking",
 }
 
+# Script (data transform) invoke options
+SCRIPT_USE_IN_FEATURE_BATCH = "BatchTransform"
+SCRIPT_USE_IN_FEATURE_STREAMING = "StreamingTransform"
+SCRIPT_USE_IN_FEATURE_OPTIONS = [
+    SCRIPT_USE_IN_FEATURE_BATCH,
+    SCRIPT_USE_IN_FEATURE_STREAMING,
+]
+
 # Pydantic request/response type names to feature names
 REQUEST_TYPE_TO_FEATURE = {
     "SearchIndexChunkingV1Request": "SearchIndexChunking",

{salesforce_data_customcode-6.0.6.dev2 → salesforce_data_customcode-6.0.6.dev3}/src/datacustomcode/deploy.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 from __future__ import annotations
 
+import copy
 from html import unescape
 import json
 import os
@@ -41,6 +42,7 @@ from datacustomcode.scan import find_base_directory, get_package_type
 
 DATA_CUSTOM_CODE_PATH = "services/data/v63.0/ssot/data-custom-code"
 DATA_TRANSFORMS_PATH = "services/data/v63.0/ssot/data-transforms"
+DATA_CUSTOM_CODE_INVOKE_OPTIONS_PATH = "services/data/v67.0/ssot/data-custom-code"
 WAIT_FOR_DEPLOYMENT_TIMEOUT = 3000
 
 # Available compute types for Data Cloud deployments.
@@ -108,6 +110,7 @@ class CodeExtensionMetadata(BaseModel):
     computeType: str
     codeType: str
     functionInvokeOptions: Union[list[str], None] = None
+    invokeOptions: Union[list[str], None] = None
 
     def __init__(self, **data):
         name = data.get("name", "")
@@ -200,7 +203,14 @@ def create_deployment(
     access_token: AccessTokenResponse, metadata: CodeExtensionMetadata
 ) -> CreateDeploymentResponse:
     """Create a custom code deployment in the DataCloud."""
-    url = _join_strip_url(access_token.instance_url, DATA_CUSTOM_CODE_PATH)
+    # invokeOptions only binds at v67.0; route there when it is set so the
+    # option isn't silently dropped. Everything else stays on v63.0.
+    code_custom_code_path = (
+        DATA_CUSTOM_CODE_INVOKE_OPTIONS_PATH
+        if metadata.invokeOptions
+        else DATA_CUSTOM_CODE_PATH
+    )
+    url = _join_strip_url(access_token.instance_url, code_custom_code_path)
     body = dict[str, Any](
         {
             "label": metadata.name,
@@ -213,6 +223,8 @@ def create_deployment(
     )
     if metadata.functionInvokeOptions:
         body["functionInvokeOptions"] = metadata.functionInvokeOptions
+    if metadata.invokeOptions:
+        body["invokeOptions"] = metadata.invokeOptions
     logger.debug(f"Creating deployment {metadata.name}...")
     try:
         response = _make_api_call(
@@ -388,6 +400,30 @@ class DataTransformConfig(BaseConfig):
     dataspace: str
     permissions: Permissions
     dataObjects: Optional[list[DataObject]] = None
+    streamingSource: Optional[StreamingSource] = None
+
+    @property
+    def is_streaming(self) -> bool:
+        return self.streamingSource is not None
+
+    @model_validator(mode="after")
+    def _validate_layers(self) -> "DataTransformConfig":
+        read_is_dlo = isinstance(self.permissions.read, DloPermission)
+        write_is_dlo = isinstance(self.permissions.write, DloPermission)
+        if self.is_streaming:
+            if not write_is_dlo:
+                raise ValueError(
+                    "A streaming transform must write to a DLO "
+                    "(permissions.write must be a 'dlo' entry)."
+                )
+        elif read_is_dlo != write_is_dlo:
+            raise ValueError(
+                "permissions.read and permissions.write must both reference "
+                "DLOs or both reference DMOs (got "
+                f"read={type(self.permissions.read).__name__}, "
+                f"write={type(self.permissions.write).__name__})"
+            )
+        return self
 
 
 class FunctionConfig(BaseConfig):
@@ -402,23 +438,15 @@ class DmoPermission(BaseModel):
     dmo: list[str]
 
 
+class StreamingSource(BaseModel):
+    type: str
+    name: str
+
+
 class Permissions(BaseModel):
     read: Union[DloPermission, DmoPermission]
     write: Union[DloPermission, DmoPermission]
 
-    @model_validator(mode="after")
-    def _no_mixed_layers(self) -> "Permissions":
-        read_is_dlo = isinstance(self.read, DloPermission)
-        write_is_dlo = isinstance(self.write, DloPermission)
-        if read_is_dlo != write_is_dlo:
-            raise ValueError(
-                "permissions.read and permissions.write must both reference "
-                "DLOs or both reference DMOs (got "
-                f"read={type(self.read).__name__}, "
-                f"write={type(self.write).__name__})"
-            )
-        return self
-
 
 def _permission_entries(perm: Union[DloPermission, DmoPermission]) -> list[str]:
     """Return the list of object names regardless of layer (DLO or DMO)."""
@@ -490,7 +518,9 @@ def create_data_transform(
 ) -> dict:
     """Create a data transform in the DataCloud."""
     script_name = metadata.name
-    request_hydrated = DATA_TRANSFORM_REQUEST_TEMPLATE.copy()
+    # Deep copy: the template's nested nodes/sources/macros dicts would
+    # otherwise be shared across calls and accumulate entries between deploys.
+    request_hydrated = copy.deepcopy(DATA_TRANSFORM_REQUEST_TEMPLATE)
 
     # Add nodes for each write entry (DLO or DMO)
     for i, name in enumerate(
@@ -533,7 +563,7 @@ def create_data_transform(
         "definition": definition,
         "label": f"{metadata.name}",
         "name": f"{metadata.name}",
-        "type": "BATCH",
+        "type": "STREAMING" if data_transform_config.is_streaming else "BATCH",
         "dataSpaceName": data_transform_config.dataspace,
     }
 
@@ -616,9 +646,18 @@ def deploy_full(
     callback=None,
 ) -> AccessTokenResponse:
     """Deploy a data transform in the DataCloud."""
+    from datacustomcode.constants import SCRIPT_USE_IN_FEATURE_STREAMING
+
     # prepare payload
     config = get_config(directory)
 
+    if (
+        isinstance(config, DataTransformConfig)
+        and config.is_streaming
+        and not metadata.invokeOptions
+    ):
+        metadata.invokeOptions = [SCRIPT_USE_IN_FEATURE_STREAMING]
+
     # create deployment and upload payload
     deployment = create_deployment(access_token, metadata)
     zip(directory, docker_network, metadata.codeType)

{salesforce_data_customcode-6.0.6.dev2 → salesforce_data_customcode-6.0.6.dev3}/src/datacustomcode/scan.py

@@ -15,6 +15,7 @@
 from __future__ import annotations
 
 import ast
+import copy
 import json
 import os
 import sys
@@ -32,6 +33,8 @@ import pydantic
 from datacustomcode.version import get_version
 
 DATA_ACCESS_METHODS = ["read_dlo", "read_dmo", "write_to_dlo", "write_to_dmo"]
+STREAMING_READ_METHODS = ["read_dlo_deltas", "read_dmo_deltas"]
+STREAMING_WRITE_METHODS = ["write_dlo_deltas"]
 
 DATA_TRANSFORM_CONFIG_TEMPLATE = {
     "sdkVersion": get_version(),
@@ -43,6 +46,20 @@ DATA_TRANSFORM_CONFIG_TEMPLATE = {
     },
 }
 
+STREAMING_TRANSFORM_CONFIG_TEMPLATE = {
+    "sdkVersion": get_version(),
+    "entryPoint": "",
+    "dataspace": "default",
+    "streamingSource": {
+        "type": "dlo",
+        "name": "",
+    },
+    "permissions": {
+        "read": {},
+        "write": {},
+    },
+}
+
 FUNCTION_CONFIG_TEMPLATE = {
     "entryPoint": "",
 }
@@ -160,6 +177,35 @@ class DataAccessLayerCalls(pydantic.BaseModel):
         return next(iter(self.write_to_dmo))
 
 
+class StreamingDataAccessLayerCalls(pydantic.BaseModel):
+    read_dlo_deltas: bool
+    read_dmo_deltas: bool
+    write_dlo_deltas: frozenset[str]
+
+    @pydantic.model_validator(mode="after")
+    def validate_access_layer(self) -> StreamingDataAccessLayerCalls:
+        if self.read_dlo_deltas and self.read_dmo_deltas:
+            raise ValueError(
+                "Cannot read DLO and DMO deltas in the same streaming transform."
+            )
+        if not self.read_dlo_deltas and not self.read_dmo_deltas:
+            raise ValueError(
+                "A streaming transform must read from at least one DLO or DMO "
+                "delta stream (read_dlo_deltas / read_dmo_deltas)."
+            )
+        if not self.write_dlo_deltas:
+            raise ValueError(
+                "A streaming transform must write to at least one DLO via "
+                "write_dlo_deltas."
+            )
+        return self
+
+    @property
+    def read_layer(self) -> str:
+        """Return the read source layer, ``"dlo"`` or ``"dmo"``."""
+        return "dlo" if self.read_dlo_deltas else "dmo"
+
+
 class ClientMethodVisitor(ast.NodeVisitor):
     """AST Visitor that finds all instances of Client read/write method calls."""
 
@@ -168,6 +214,9 @@ class ClientMethodVisitor(ast.NodeVisitor):
         self._read_dmo_instances: set[str] = set()
         self._write_to_dlo_instances: set[str] = set()
         self._write_to_dmo_instances: set[str] = set()
+        self._read_dlo_deltas: bool = False
+        self._read_dmo_deltas: bool = False
+        self._write_dlo_deltas_instances: set[str] = set()
         self.variable_values: Dict[str, Union[str, None]] = {}
 
     def visit_Assign(self, node: ast.Assign) -> None:
@@ -189,14 +238,15 @@ class ClientMethodVisitor(ast.NodeVisitor):
             node.func.value, ast.Name
         ):
             method_name = node.func.attr
+
+            if method_name == "read_dlo_deltas":
+                self._read_dlo_deltas = True
+            elif method_name == "read_dmo_deltas":
+                self._read_dmo_deltas = True
+
             if method_name in DATA_ACCESS_METHODS and node.args:
                 arg = node.args[0]
-                name = None
-
-                if isinstance(arg, ast.Constant) and isinstance(arg.value, str):
-                    name = arg.value
-                elif isinstance(arg, ast.Name) and arg.id in self.variable_values:
-                    name = self.variable_values[arg.id]
+                name = self._resolve_name_arg(arg)
 
                 if name:
                     if method_name == "read_dlo":
@@ -207,8 +257,29 @@ class ClientMethodVisitor(ast.NodeVisitor):
                         self._write_to_dlo_instances.add(name)
                     elif method_name == "write_to_dmo":
                         self._write_to_dmo_instances.add(name)
+            elif method_name in STREAMING_WRITE_METHODS and node.args:
+                name = self._resolve_name_arg(node.args[0])
+                if name and method_name == "write_dlo_deltas":
+                    self._write_dlo_deltas_instances.add(name)
         self.generic_visit(node)
 
+    def _resolve_name_arg(self, arg: ast.expr) -> Union[str, None]:
+        """Resolve a string-literal or tracked-variable first argument."""
+        if isinstance(arg, ast.Constant) and isinstance(arg.value, str):
+            return arg.value
+        if isinstance(arg, ast.Name) and arg.id in self.variable_values:
+            return self.variable_values[arg.id]
+        return None
+
+    @property
+    def is_streaming(self) -> bool:
+        """Whether any streaming (delta) access method was found."""
+        return (
+            self._read_dlo_deltas
+            or self._read_dmo_deltas
+            or bool(self._write_dlo_deltas_instances)
+        )
+
     def found(self) -> DataAccessLayerCalls:
         return DataAccessLayerCalls(
             read_dlo=frozenset(self._read_dlo_instances),
@@ -217,6 +288,13 @@ class ClientMethodVisitor(ast.NodeVisitor):
             write_to_dmo=frozenset(self._write_to_dmo_instances),
         )
 
+    def found_streaming(self) -> StreamingDataAccessLayerCalls:
+        return StreamingDataAccessLayerCalls(
+            read_dlo_deltas=self._read_dlo_deltas,
+            read_dmo_deltas=self._read_dmo_deltas,
+            write_dlo_deltas=frozenset(self._write_dlo_deltas_instances),
+        )
+
 
 class ImportVisitor(ast.NodeVisitor):
     """AST Visitor that extracts external package imports from Python code."""
@@ -301,23 +379,51 @@ def write_requirements_file(file_path: str) -> str:
     return requirements_path
 
 
-def scan_file(file_path: str) -> DataAccessLayerCalls:
-    """Scan a single Python file for Client read/write method calls."""
+def _visit_file(file_path: str) -> ClientMethodVisitor:
+    """Parse a Python file and return the populated method visitor."""
     with open(file_path, "r") as f:
-        code = f.read()
-        tree = ast.parse(code)
-        visitor = ClientMethodVisitor()
-        visitor.visit(tree)
-        return visitor.found()
+        tree = ast.parse(f.read())
+    visitor = ClientMethodVisitor()
+    visitor.visit(tree)
+    return visitor
+
+
+def scan_file(file_path: str) -> DataAccessLayerCalls:
+    """Scan a single Python file for batch Client read/write method calls."""
+    return _visit_file(file_path).found()
+
+
+def scan_file_streaming(file_path: str) -> StreamingDataAccessLayerCalls:
+    """Scan a single Python file for StreamingClient delta method calls."""
+    return _visit_file(file_path).found_streaming()
+
 
+def file_is_streaming(file_path: str) -> bool:
+    """Return whether the entrypoint uses streaming (delta) access methods."""
+    return _visit_file(file_path).is_streaming
 
-def dc_config_json_from_file(file_path: str, type: str) -> dict[str, Any]:
-    """Create a Data Cloud Custom Code config JSON from a script."""
+
+def dc_config_json_from_file(
+    file_path: str, type: str, streaming: bool = False
+) -> dict[str, Any]:
+    """Create a Data Cloud Custom Code config JSON from a script.
+
+    Args:
+        file_path: Path to the entrypoint.
+        type: Package type, ``"script"`` or ``"function"``.
+        streaming: For scripts, a streaming
+            (``streamingSource``) config instead of a batch one.
+    """
     config: dict[str, Any]
     if type == "script":
-        config = DATA_TRANSFORM_CONFIG_TEMPLATE.copy()
+        template = (
+            STREAMING_TRANSFORM_CONFIG_TEMPLATE
+            if streaming
+            else DATA_TRANSFORM_CONFIG_TEMPLATE
+        )
+        config = copy.deepcopy(template)
     elif type == "function":
-        config = FUNCTION_CONFIG_TEMPLATE.copy()
+        config = copy.deepcopy(FUNCTION_CONFIG_TEMPLATE)
     config["entryPoint"] = os.path.basename(file_path)
     return config
 
@@ -372,22 +478,51 @@ def update_config(file_path: str) -> dict[str, Any]:
 
     if package_type == "script":
         existing_config["dataspace"] = get_dataspace(existing_config)
-        output = scan_file(file_path)
-        read: dict[str, list[str]] = {}
-        if output.read_dlo:
-            read["dlo"] = list(output.read_dlo)
-        else:
-            read["dmo"] = list(output.read_dmo)
-        write: dict[str, list[str]] = {}
-        if output.write_to_dlo:
-            write["dlo"] = list(output.write_to_dlo)
+        if file_is_streaming(file_path):
+            _update_streaming_config(existing_config, file_path)
         else:
-            write["dmo"] = list(output.write_to_dmo)
-
-        existing_config["permissions"] = {"read": read, "write": write}
+            existing_config.pop("streamingSource", None)
+            output = scan_file(file_path)
+            read: dict[str, list[str]] = {}
+            if output.read_dlo:
+                read["dlo"] = list(output.read_dlo)
+            else:
+                read["dmo"] = list(output.read_dmo)
+            write: dict[str, list[str]] = {}
+            if output.write_to_dlo:
+                write["dlo"] = list(output.write_to_dlo)
+            else:
+                write["dmo"] = list(output.write_to_dmo)
+
+            existing_config["permissions"] = {"read": read, "write": write}
     return existing_config
 
 
+def _update_streaming_config(existing_config: dict[str, Any], file_path: str) -> None:
+    output = scan_file_streaming(file_path)
+    read_layer = output.read_layer
+
+    source = existing_config.get("streamingSource")
+    if not isinstance(source, dict):
+        source = {}
+    source_name = source.get("name", "")
+    existing_config["streamingSource"] = {"type": read_layer, "name": source_name}
+
+    if not source_name:
+        logger.warning(
+            "streamingSource.name is empty in config.json. A streaming "
+            "transform must declare its read source; set streamingSource.name "
+            "to the DLO/DMO the transform reads from."
+        )
+
+    read_names = [source_name] if source_name else []
+    write_names = list(output.write_dlo_deltas)
+    existing_config["permissions"] = {
+        "read": {read_layer: read_names},
+        "write": {"dlo": write_names},
+    }
+
+
 def get_dataspace(existing_config: dict[str, str]) -> str:
     if "dataspace" in existing_config:
         dataspace_value = existing_config["dataspace"]

{salesforce_data_customcode-6.0.6.dev2 → salesforce_data_customcode-6.0.6.dev3}/src/datacustomcode/template.py

@@ -23,8 +23,12 @@ from datacustomcode.constants import FEATURE_TEMPLATE_MAPPING
 script_template_dir = os.path.join(os.path.dirname(__file__), "templates", "script")
 function_template_dir = os.path.join(os.path.dirname(__file__), "templates", "function")
 
+STREAMING_EXAMPLE_ENTRYPOINT = os.path.join(
+    script_template_dir, "examples", "streaming_deltas", "entrypoint.py"
+)
 
-def copy_script_template(target_dir: str) -> None:
+
+def copy_script_template(target_dir: str, streaming: bool = False) -> None:
     """Copy the template to the target directory."""
     os.makedirs(target_dir, exist_ok=True)
 
@@ -39,6 +43,14 @@ def copy_script_template(target_dir: str) -> None:
             logger.debug(f"Copying file {source} to {destination}...")
             shutil.copy2(source, destination)
 
+    if streaming:
+        destination = os.path.join(target_dir, "payload", "entrypoint.py")
+        logger.debug(
+            f"Copying streaming example {STREAMING_EXAMPLE_ENTRYPOINT} to "
+            f"{destination}..."
+        )
+        shutil.copy2(STREAMING_EXAMPLE_ENTRYPOINT, destination)
+
 
 def copy_function_template(target_dir: str, use_in_feature: Optional[str]) -> None:
     os.makedirs(target_dir, exist_ok=True)