salesforce-data-customcode 7.0.0rc4__tar.gz → 7.0.0rc5__tar.gz

{salesforce_data_customcode-7.0.0rc4 → salesforce_data_customcode-7.0.0rc5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: salesforce-data-customcode
-Version: 7.0.0rc4
+Version: 7.0.0rc5
 Summary: Data Cloud Custom Code SDK
 License-Expression: Apache-2.0
 License-File: LICENSE.txt
@@ -183,6 +183,7 @@ For a streaming (delta) transform, use `StreamingClient`, which exposes the stre
 * `read_dlo_deltas()` – Read the streaming change feed (deltas) of a Data Lake Object as a streaming DataFrame.
 * `read_dmo_deltas()` – Read the streaming change feed (deltas) of a Data Model Object as a streaming DataFrame.
 * `write_dlo_deltas(name, spark_dataframe)` – Write a streaming DataFrame of deltas to a Data Lake Object; returns the started `StreamingQuery`
+* `write_dmo_deltas(name, spark_dataframe)` – Write a streaming DataFrame of deltas to a Data Model Object; returns the started `StreamingQuery`
 
 `find_file_path`, `llm_gateway_generate_text`, and `einstein_predict` are available on both clients.
 
@@ -247,6 +248,15 @@ query = client.write_dlo_deltas("Output__dll", transformed)
 query.awaitTermination()
 ```
 
+A streaming transform can also read and write Data Model Objects with `read_dmo_deltas()` and `write_dmo_deltas()`:
+
+```python
+deltas = client.read_dmo_deltas()
+transformed = deltas.withColumn("description__c", upper(col("description__c")))
+query = client.write_dmo_deltas("Output__dlm", transformed)
+query.awaitTermination()
+```
+
 Notes:
 
 - These methods only run inside the Data Cloud streaming (`DELTA_SYNC`) runtime. Locally (`datacustomcode run`) they raise `NotImplementedError`, since there is no change feed to stream.
@@ -408,6 +418,8 @@ Options:
 #### `datacustomcode zip`
 Zip a transformation job in preparation to upload to Data Cloud. Make sure to change directory into your code package folder (e.g., `my_package`) before running this command.
 
+Symlinks inside the payload folder are not followed or packaged. Each symlinked file that is skipped is reported with a warning, and symlinked directories are not included. `datacustomcode deploy` zips the same way.
+
 Arguments:
 - `PATH`: Path to the code directory i.e. the payload folder (default: "payload")
 

{salesforce_data_customcode-7.0.0rc4 → salesforce_data_customcode-7.0.0rc5}/README.md

@@ -158,6 +158,7 @@ For a streaming (delta) transform, use `StreamingClient`, which exposes the stre
 * `read_dlo_deltas()` – Read the streaming change feed (deltas) of a Data Lake Object as a streaming DataFrame.
 * `read_dmo_deltas()` – Read the streaming change feed (deltas) of a Data Model Object as a streaming DataFrame.
 * `write_dlo_deltas(name, spark_dataframe)` – Write a streaming DataFrame of deltas to a Data Lake Object; returns the started `StreamingQuery`
+* `write_dmo_deltas(name, spark_dataframe)` – Write a streaming DataFrame of deltas to a Data Model Object; returns the started `StreamingQuery`
 
 `find_file_path`, `llm_gateway_generate_text`, and `einstein_predict` are available on both clients.
 
@@ -222,6 +223,15 @@ query = client.write_dlo_deltas("Output__dll", transformed)
 query.awaitTermination()
 ```
 
+A streaming transform can also read and write Data Model Objects with `read_dmo_deltas()` and `write_dmo_deltas()`:
+
+```python
+deltas = client.read_dmo_deltas()
+transformed = deltas.withColumn("description__c", upper(col("description__c")))
+query = client.write_dmo_deltas("Output__dlm", transformed)
+query.awaitTermination()
+```
+
 Notes:
 
 - These methods only run inside the Data Cloud streaming (`DELTA_SYNC`) runtime. Locally (`datacustomcode run`) they raise `NotImplementedError`, since there is no change feed to stream.
@@ -383,6 +393,8 @@ Options:
 #### `datacustomcode zip`
 Zip a transformation job in preparation to upload to Data Cloud. Make sure to change directory into your code package folder (e.g., `my_package`) before running this command.
 
+Symlinks inside the payload folder are not followed or packaged. Each symlinked file that is skipped is reported with a warning, and symlinked directories are not included. `datacustomcode deploy` zips the same way.
+
 Arguments:
 - `PATH`: Path to the code directory i.e. the payload folder (default: "payload")
 

{salesforce_data_customcode-7.0.0rc4 → salesforce_data_customcode-7.0.0rc5}/pyproject.toml

@@ -18,7 +18,7 @@ license = "Apache-2.0"
 name = "salesforce-data-customcode"
 readme = "README.md"
 requires-python = ">=3.10,<3.12"
-version = "7.0.0rc4"
+version = "7.0.0rc5"
 
 [tool.black]
 exclude = '''

{salesforce_data_customcode-7.0.0rc4 → salesforce_data_customcode-7.0.0rc5}/src/datacustomcode/client.py

@@ -726,6 +726,25 @@ class StreamingClient(_BaseClient):
         self._validate_data_layer_history_does_not_contain(DataCloudObjectType.DMO)
         return self._writer.write_dlo_deltas(name, dataframe, **kwargs)  # type: ignore[no-any-return]
 
+    def write_dmo_deltas(
+        self, name: str, dataframe: PySparkDataFrame, **kwargs
+    ) -> StreamingQuery:
+        """Write a streaming DataFrame of deltas to a DMO in Data Cloud.
+
+        Starts a streaming query that writes each micro-batch to the
+        target DMO and returns the ``StreamingQuery`` handle; the caller
+        typically calls ``query.awaitTermination()``.
+
+        Args:
+            name: The name of the DMO to write to.
+            dataframe: The streaming PySpark DataFrame to write.
+
+        Returns:
+            The started ``StreamingQuery``.
+        """
+        self._validate_data_layer_history_does_not_contain(DataCloudObjectType.DLO)
+        return self._writer.write_dmo_deltas(name, dataframe, **kwargs)  # type: ignore[no-any-return]
+
     def auto_write_to_dlo(self, name: str, dataframe: PySparkDataFrame) -> None:
         """Write a PySpark DataFrame to a DLO in Data Cloud automatically picking
         the WriteMode.

{salesforce_data_customcode-7.0.0rc4 → salesforce_data_customcode-7.0.0rc5}/src/datacustomcode/deploy.py

@@ -415,13 +415,7 @@ class DataTransformConfig(BaseConfig):
     def _validate_layers(self) -> "DataTransformConfig":
         read_is_dlo = isinstance(self.permissions.read, DloPermission)
         write_is_dlo = isinstance(self.permissions.write, DloPermission)
-        if self.is_streaming:
-            if not write_is_dlo:
-                raise ValueError(
-                    "A streaming transform must write to a DLO "
-                    "(permissions.write must be a 'dlo' entry)."
-                )
-        elif read_is_dlo != write_is_dlo:
+        if read_is_dlo != write_is_dlo:
             raise ValueError(
                 "permissions.read and permissions.write must both reference "
                 "DLOs or both reference DMOs (got "
@@ -630,12 +624,15 @@ def zip(
     logger.debug(f"Zipping directory... {directory}")
 
     with zipfile.ZipFile(ZIP_FILE_NAME, "w", zipfile.ZIP_DEFLATED) as zipf:
-        for root, dirs, files in os.walk(directory):
-            # Skip .DS_Store and local credentials.
+        for root, dirs, files in os.walk(directory, followlinks=False):
+            # Skip symlinks, .DS_Store and local credentials.
             for file in files:
+                abs_path = os.path.join(root, file)
+                arcname = os.path.relpath(abs_path, directory)
+                if os.path.islink(abs_path):
+                    logger.warning(f"Skipping symlink file {arcname}")
+                    continue
                 if file not in EXCLUDED_FILES:
-                    abs_path = os.path.join(root, file)
-                    arcname = os.path.relpath(abs_path, directory)
                     zipf.write(abs_path, arcname)
 
     logger.debug(f"Created zip file: {ZIP_FILE_NAME}")

{salesforce_data_customcode-7.0.0rc4 → salesforce_data_customcode-7.0.0rc5}/src/datacustomcode/io/writer/base.py

@@ -87,7 +87,7 @@ class BaseDataCloudWriter(BaseDataAccessLayer):
         Args:
             name: Target Data Lake Object name.
             dataframe: Streaming PySpark DataFrame produced from a
-                ``read_dlo_deltas`` / ``read_dmo_deltas`` source.
+                ``read_dlo_deltas`` source.
 
         Returns:
             The started ``StreamingQuery``; the caller drives its lifecycle
@@ -102,3 +102,35 @@ class BaseDataCloudWriter(BaseDataAccessLayer):
             "streaming runtime; the local writer does not support streaming "
             "deltas."
         )
+
+    def write_dmo_deltas(
+        self, name: str, dataframe: PySparkDataFrame
+    ) -> StreamingQuery:
+        """Write a streaming DataFrame of deltas to a Data Model Object.
+
+        Streaming counterpart to :meth:`write_to_dmo`. Starts a streaming query
+        that writes each micro-batch to the target DMO via the Data Cloud
+        streaming sink and returns the resulting ``StreamingQuery`` handle. The
+        runtime owns the trigger and checkpoint location; callers pass only the
+        table name. Concrete streaming behavior is provided by the deployed
+        Data Cloud runtime; the base implementation raises
+        :class:`NotImplementedError`.
+
+        Args:
+            name: Target Data Model Object name.
+            dataframe: Streaming PySpark DataFrame produced from a
+                ``read_dmo_deltas`` source.
+
+        Returns:
+            The started ``StreamingQuery``; the caller drives its lifecycle
+            (typically ``query.awaitTermination()``).
+
+        Raises:
+            NotImplementedError: If the active writer does not support streaming
+                deltas (e.g. the local development writers).
+        """
+        raise NotImplementedError(
+            "write_dmo_deltas is only supported when running in the Data Cloud "
+            "streaming runtime; the local writer does not support streaming "
+            "deltas."
+        )

{salesforce_data_customcode-7.0.0rc4 → salesforce_data_customcode-7.0.0rc5}/src/datacustomcode/scan.py

@@ -34,7 +34,7 @@ from datacustomcode.version import get_version
 
 DATA_ACCESS_METHODS = ["read_dlo", "read_dmo", "write_to_dlo", "write_to_dmo"]
 STREAMING_READ_METHODS = ["read_dlo_deltas", "read_dmo_deltas"]
-STREAMING_WRITE_METHODS = ["write_dlo_deltas"]
+STREAMING_WRITE_METHODS = ["write_dlo_deltas", "write_dmo_deltas"]
 
 DATA_TRANSFORM_CONFIG_TEMPLATE = {
     "sdkVersion": get_version(),
@@ -181,6 +181,7 @@ class StreamingDataAccessLayerCalls(pydantic.BaseModel):
     read_dlo_deltas: bool
     read_dmo_deltas: bool
     write_dlo_deltas: frozenset[str]
+    write_dmo_deltas: frozenset[str]
 
     @pydantic.model_validator(mode="after")
     def validate_access_layer(self) -> StreamingDataAccessLayerCalls:
@@ -193,10 +194,19 @@ class StreamingDataAccessLayerCalls(pydantic.BaseModel):
                 "A streaming transform must read from at least one DLO or DMO "
                 "delta stream (read_dlo_deltas / read_dmo_deltas)."
             )
-        if not self.write_dlo_deltas:
+        if not self.write_dlo_deltas and not self.write_dmo_deltas:
             raise ValueError(
-                "A streaming transform must write to at least one DLO via "
-                "write_dlo_deltas."
+                "A streaming transform must write to at least one DLO or DMO "
+                "via write_dlo_deltas / write_dmo_deltas."
+            )
+        if self.write_dlo_deltas and self.write_dmo_deltas:
+            raise ValueError(
+                "Cannot write DLO and DMO deltas in the same streaming transform."
+            )
+        if self.read_layer != self.write_layer:
+            raise ValueError(
+                "A streaming transform must read and write the same layer "
+                f"(got read={self.read_layer}, write={self.write_layer})."
             )
         return self
 
@@ -205,6 +215,11 @@ class StreamingDataAccessLayerCalls(pydantic.BaseModel):
         """Return the read source layer, ``"dlo"`` or ``"dmo"``."""
         return "dlo" if self.read_dlo_deltas else "dmo"
 
+    @property
+    def write_layer(self) -> str:
+        """Return the write target layer, ``"dlo"`` or ``"dmo"``."""
+        return "dlo" if self.write_dlo_deltas else "dmo"
+
 
 class ClientMethodVisitor(ast.NodeVisitor):
     """AST Visitor that finds all instances of Client read/write method calls."""
@@ -217,6 +232,7 @@ class ClientMethodVisitor(ast.NodeVisitor):
         self._read_dlo_deltas: bool = False
         self._read_dmo_deltas: bool = False
         self._write_dlo_deltas_instances: set[str] = set()
+        self._write_dmo_deltas_instances: set[str] = set()
         self.variable_values: Dict[str, Union[str, None]] = {}
 
     def visit_Assign(self, node: ast.Assign) -> None:
@@ -261,6 +277,8 @@ class ClientMethodVisitor(ast.NodeVisitor):
                 name = self._resolve_name_arg(node.args[0])
                 if name and method_name == "write_dlo_deltas":
                     self._write_dlo_deltas_instances.add(name)
+                elif name and method_name == "write_dmo_deltas":
+                    self._write_dmo_deltas_instances.add(name)
         self.generic_visit(node)
 
     def _resolve_name_arg(self, arg: ast.expr) -> Union[str, None]:
@@ -278,6 +296,7 @@ class ClientMethodVisitor(ast.NodeVisitor):
             self._read_dlo_deltas
             or self._read_dmo_deltas
             or bool(self._write_dlo_deltas_instances)
+            or bool(self._write_dmo_deltas_instances)
         )
 
     def found(self) -> DataAccessLayerCalls:
@@ -293,6 +312,7 @@ class ClientMethodVisitor(ast.NodeVisitor):
             read_dlo_deltas=self._read_dlo_deltas,
             read_dmo_deltas=self._read_dmo_deltas,
             write_dlo_deltas=frozenset(self._write_dlo_deltas_instances),
+            write_dmo_deltas=frozenset(self._write_dmo_deltas_instances),
         )
 
 
@@ -516,10 +536,10 @@ def _update_streaming_config(existing_config: dict[str, Any], file_path: str) ->
         )
 
     read_names = [source_name] if source_name else []
-    write_names = list(output.write_dlo_deltas)
+    write_names = list(output.write_dlo_deltas or output.write_dmo_deltas)
     existing_config["permissions"] = {
         "read": {read_layer: read_names},
-        "write": {"dlo": write_names},
+        "write": {output.write_layer: write_names},
     }