salesforce-data-customcode 6.1.0.dev4__py3-none-any.whl → 6.1.0.dev5__py3-none-any.whl

datacustomcode/client.py

@@ -15,6 +15,7 @@
 from __future__ import annotations
 
 from enum import Enum
+import os
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -31,7 +32,6 @@ from datacustomcode.einstein_predictions_config import spark_einstein_prediction
 from datacustomcode.file.path.default import DefaultFindFilePath
 from datacustomcode.io.reader.base import BaseDataCloudReader
 from datacustomcode.llm_gateway_config import spark_llm_gateway_config
-from datacustomcode.named_credential_config import spark_named_credential_config
 from datacustomcode.spark.default import DefaultSparkSessionProvider
 
 if TYPE_CHECKING:
@@ -49,9 +49,6 @@ if TYPE_CHECKING:
     from datacustomcode.io.reader.base import BaseDataCloudReader
     from datacustomcode.io.writer.base import BaseDataCloudWriter, WriteMode
     from datacustomcode.llm_gateway.spark_base import SparkLLMGateway
-    from datacustomcode.named_credential.spark_base import SparkNamedCredential
-    from datacustomcode.named_credential.types.http_request import HTTPRequest
-    from datacustomcode.named_credential.types.http_response import HTTPResponse
     from datacustomcode.spark.base import BaseSparkSessionProvider
 
 
@@ -163,21 +160,6 @@ def _build_spark_einstein_predictions() -> "SparkEinsteinPredictions":
     return cfg.to_object()
 
 
-def _build_spark_named_credential() -> "SparkNamedCredential":
-    """Instantiate the SDK-configured :class:`SparkNamedCredential`.
-
-    Raises:
-        RuntimeError: If no ``spark_named_credential_config`` has been loaded.
-    """
-    cfg = spark_named_credential_config.spark_named_credential_config
-    if cfg is None:
-        raise RuntimeError(
-            "spark_named_credential_config is not configured. Add a "
-            "'spark_named_credential_config' section to config.yaml."
-        )
-    return cfg.to_object()
-
-
 def einstein_predict_col(
     model_api_name: str,
     prediction_type: "PredictionType",
@@ -227,40 +209,6 @@ def einstein_predict_col(
     )
 
 
-def named_credential_request_col(
-    request: "HTTPRequest",
-    body: Optional["Column"] = None,
-) -> "Column":
-    """Build a Spark Column that makes one Named Credential callout per row.
-
-    The endpoint, method, and headers are fixed for the call (taken from
-    ``request``); only ``body`` varies per row. Use this instead of
-    :meth:`Client.named_credential_request` when the callout runs across a
-    DataFrame so each row is dispatched independently rather than one-shot on
-    the driver.
-
-    The returned Column yields a struct ``{status, response, error_code,
-    error_message}`` for each row. ``response`` is itself a struct
-    ``{status_code, body, headers}``. Use ``[...]`` to pick a field, e.g.
-    ``named_credential_request_col(...)["response"]["status_code"]``. A transport
-    failure sets ``status`` to ``ERROR`` and populates ``error_message`` (a non-2xx
-    HTTP response is still ``SUCCESS`` with its code in ``response.status_code``),
-    so a single bad row does not abort the whole Spark job.
-
-    Args:
-        request: The callout template — its symbolic reference, method, and
-            headers are applied to every row.
-        body: Optional per-row ``Column`` holding the request body as a
-            string (or null for no body).
-
-    Returns:
-        A Spark ``Column`` of ``StructType`` with fields ``status``,
-        ``response``, ``error_code``, and ``error_message``.
-    """
-    named_credential = Client()._get_spark_named_credential()
-    return named_credential.request_col(request, body=body)
-
-
 class DataCloudObjectType(Enum):
     DLO = "dlo"
     DMO = "dmo"
@@ -318,14 +266,6 @@ class _BaseClient:
         spark_llm_gateway: Optional custom :class:`SparkLLMGateway`.
         spark_einstein_predictions: Optional custom
             :class:`SparkEinsteinPredictions`.
-        spark_named_credential: Optional custom :class:`SparkNamedCredential`.
-
-    Example:
-    >>> client = Client()
-    >>> file_path = client.find_file_path("data.csv")
-    >>> dlo = client.read_dlo("my_dlo")
-    >>> client.write_to_dmo("my_dmo", dlo)
-    >>> answer = client.llm_gateway_generate_text("Generate a greeting message")
     """
 
     # Each concrete subclass gets its own ``_instance`` slot: reads fall through
@@ -345,7 +285,6 @@ class _BaseClient:
     _file: DefaultFindFilePath
     _spark_llm_gateway: Optional[SparkLLMGateway]
     _spark_einstein_predictions: Optional[SparkEinsteinPredictions]
-    _spark_named_credential: Optional[SparkNamedCredential]
     _data_layer_history: dict[DataCloudObjectType, set[str]]
     _code_type: str
 
@@ -356,7 +295,6 @@ class _BaseClient:
         spark_provider: Optional[BaseSparkSessionProvider] = None,
         spark_llm_gateway: Optional[SparkLLMGateway] = None,
         spark_einstein_predictions: Optional[SparkEinsteinPredictions] = None,
-        spark_named_credential: Optional[SparkNamedCredential] = None,
         code_type: str = "script",
     ) -> _ClientT:
 
@@ -364,7 +302,6 @@ class _BaseClient:
             instance = super().__new__(cls)
             instance._spark_llm_gateway = spark_llm_gateway
             instance._spark_einstein_predictions = spark_einstein_predictions
-            instance._spark_named_credential = spark_named_credential
             # Initialize Readers and Writers from config
             # and/or provided reader and writer
             if reader is None or writer is None:
@@ -537,41 +474,6 @@ class _BaseClient:
             self._spark_einstein_predictions = _build_spark_einstein_predictions()
         return self._spark_einstein_predictions
 
-    def named_credential_request(
-        self,
-        request: "HTTPRequest",
-        body: Optional[str] = None,
-    ) -> "HTTPResponse":
-        """Issue a one-shot Named Credential external callout. This is the
-        scalar counterpart to :func:`named_credential_request_col`: it runs
-        **once** on the driver — not per row. Use the column helper method
-        instead when you want to fan a callout out across every row of a
-        DataFrame.
-
-        Example:
-
-            >>> from datacustomcode.named_credential.types.http_request_builder \\
-            ...     import HTTPRequestBuilder
-            >>> request = (
-            ...     HTTPRequestBuilder().set_url("callout:NC/search").build()
-            ... )
-            >>> response = Client().named_credential_request(request)
-
-        Args:
-            request: The callout request
-            body: Optional request body. Set the ``Content-Type`` header to
-                match the format; the SDK does not assume or inject one.
-
-        Returns:
-            The external service's response.
-        """
-        return self._get_spark_named_credential().request(request, body=body)
-
-    def _get_spark_named_credential(self) -> SparkNamedCredential:
-        if self._spark_named_credential is None:
-            self._spark_named_credential = _build_spark_named_credential()
-        return self._spark_named_credential
-
     def _validate_data_layer_history_does_not_contain(
         self, data_cloud_object_type: DataCloudObjectType
     ) -> None:
@@ -657,6 +559,15 @@ class StreamingClient(_BaseClient):
 
     _instance: ClassVar[Optional[StreamingClient]] = None
 
+    def read_dlo(self) -> PySparkDataFrame:
+        """Read the streamingSource
+
+        Returns:
+            A standard PySpark DataFrame from the streaming source DLO
+        """
+        self._record_dlo_access(_streaming_source_name())
+        return self._reader.read_dlo(_streaming_source_name())
+
     def read_dlo_deltas(self) -> PySparkDataFrame:
         """Read the streaming change feed (deltas) for a DLO from Data Cloud.
 
@@ -670,6 +581,14 @@ class StreamingClient(_BaseClient):
         self._record_dlo_access(_streaming_source_name())
         return self._reader.read_dlo_deltas()  # type: ignore[no-any-return]
 
+    def read_dmo(self) -> PySparkDataFrame:
+        """Read the streamingSource
+
+        Returns a standard PySpark DataFrame from the streaming source DMO
+        """
+        self._record_dmo_access(_streaming_source_name())
+        return self._reader.read_dmo(_streaming_source_name())
+
     def read_dmo_deltas(self) -> PySparkDataFrame:
         """Read the streaming change feed (deltas) for a DMO from Data Cloud.
 
@@ -697,3 +616,41 @@ class StreamingClient(_BaseClient):
         """
         self._validate_data_layer_history_does_not_contain(DataCloudObjectType.DMO)
         return self._writer.write_dlo_deltas(name, dataframe, **kwargs)  # type: ignore[no-any-return]
+
+    def auto_write_to_dlo(self, name: str, dataframe: PySparkDataFrame) -> None:
+        """Write a PySpark DataFrame to a DLO in Data Cloud automatically picking
+        the WriteMode.
+        For use with streaming transforms when running in rebuild or initial sync mode.
+        Args:
+            name: The name of the DLO to write to.
+            dataframe: The PySpark DataFrame to write.
+        """
+        self._validate_data_layer_history_does_not_contain(DataCloudObjectType.DMO)
+        return self._writer.auto_write_to_dlo(name, dataframe)
+
+    def auto_write_to_dmo(self, name: str, dataframe: PySparkDataFrame) -> None:
+        """Write a PySpark DataFrame to a DMO in Data Cloud automatically picking
+        the WriteMode.
+        For use with streaming transforms when running in rebuild or initial sync mode.
+        Args:
+            name: The name of the DMO to write to.
+            dataframe: The PySpark DataFrame to write.
+        """
+        self._validate_data_layer_history_does_not_contain(DataCloudObjectType.DLO)
+        return self._writer.auto_write_to_dmo(name, dataframe)
+
+
+class RunMode(Enum):
+    BATCH = "BATCH"
+    INITIAL_SYNC = "INITIAL_SYNC"
+    REBUILD = "REBUILD"
+    DELTA_SYNC = "DELTA_SYNC"
+
+
+def get_run_mode() -> RunMode:
+    """Read and validate the BYOC_RUN_MODE env var; default to BATCH when unset."""
+    run_mode = os.getenv("BYOC_RUN_MODE", "BATCH").upper()
+    try:
+        return RunMode(run_mode)
+    except ValueError as exc:
+        raise ValueError("Set BYOC_RUN_MODE to a valid value") from exc

datacustomcode/config.yaml

@@ -34,9 +34,3 @@ llm_gateway_config:
 
 spark_llm_gateway_config:
   type_config_name: DefaultSparkLLMGateway
-
-named_credential_config:
-  type_config_name: DefaultNamedCredential
-
-spark_named_credential_config:
-  type_config_name: DefaultSparkNamedCredential

datacustomcode/deploy.py

@@ -38,9 +38,6 @@ import requests
 
 from datacustomcode.cmd import cmd_output
 from datacustomcode.constants import REQUEST_TYPE_TO_FEATURE
-from datacustomcode.named_credential.direct.credentials import (
-    EXTERNAL_CALLOUT_CREDENTIAL,
-)
 from datacustomcode.scan import find_base_directory, get_package_type
 
 DATA_CUSTOM_CODE_PATH = "services/data/v63.0/ssot/data-custom-code"
@@ -251,8 +248,6 @@ DEPENDENCIES_ARCHIVE_PATH = os.path.join(
 )
 PY_FILES_PATH = os.path.join("payload", "py-files")
 ZIP_FILE_NAME = "deployment.zip"
-# Local-only files that must never be packaged into the deployment zip.
-EXCLUDED_FILES = (".DS_Store", EXTERNAL_CALLOUT_CREDENTIAL)
 
 
 def prepare_dependency_archive(
@@ -633,9 +628,9 @@ def zip(
 
     with zipfile.ZipFile(ZIP_FILE_NAME, "w", zipfile.ZIP_DEFLATED) as zipf:
         for root, dirs, files in os.walk(directory):
-            # Skip .DS_Store and local credentials.
+            # Skip .DS_Store files when adding to zip
             for file in files:
-                if file not in EXCLUDED_FILES:
+                if file != ".DS_Store":
                     abs_path = os.path.join(root, file)
                     arcname = os.path.relpath(abs_path, directory)
                     zipf.write(abs_path, arcname)

datacustomcode/function/runtime.py

@@ -23,8 +23,6 @@ from datacustomcode.file.path.default import DefaultFindFilePath
 from datacustomcode.function.base import BaseRuntime
 from datacustomcode.llm_gateway.base import LLMGateway
 from datacustomcode.llm_gateway_config import llm_gateway_config
-from datacustomcode.named_credential.base import NamedCredential
-from datacustomcode.named_credential_config import named_credential_config
 
 
 class Runtime(BaseRuntime):
@@ -71,7 +69,6 @@ class Runtime(BaseRuntime):
         self._llm_gateway: Optional[LLMGateway] = None
         self._file = DefaultFindFilePath()
         self._einstein_predictions: Optional[EinsteinPredictions] = None
-        self._named_credential: Optional[NamedCredential] = None
 
     @property
     def llm_gateway(self) -> LLMGateway:
@@ -101,16 +98,3 @@ class Runtime(BaseRuntime):
                 einstein_predictions_config.einstein_predictions_config.to_object()
             )
         return self._einstein_predictions
-
-    @property
-    def named_credential(self) -> NamedCredential:
-        if self._named_credential is None:
-            if named_credential_config.named_credential_config is None:
-                raise RuntimeError(
-                    "Named Credential is not configured. Add "
-                    "'named_credential_config' section to config.yaml"
-                )
-            self._named_credential = (
-                named_credential_config.named_credential_config.to_object()
-            )
-        return self._named_credential

datacustomcode/io/writer/base.py

@@ -59,6 +59,18 @@ class BaseDataCloudWriter(BaseDataAccessLayer):
         self, name: str, dataframe: PySparkDataFrame, write_mode: WriteMode
     ) -> None: ...
 
+    def auto_write_to_dlo(self, name: str, dataframe: PySparkDataFrame) -> None:
+        """Write to a DLO automatically picking the write mode.
+        For use with streaming transforms when running in rebuild or initial sync mode.
+        """
+        raise NotImplementedError
+
+    def auto_write_to_dmo(self, name: str, dataframe: PySparkDataFrame) -> None:
+        """Write to a DMO automatically picking the write mode.
+        For use with streaming transforms when running in rebuild or initial sync mode.
+        """
+        raise NotImplementedError
+
     def write_dlo_deltas(
         self, name: str, dataframe: PySparkDataFrame
     ) -> StreamingQuery:

datacustomcode/io/writer/csv.py

@@ -36,6 +36,10 @@ class CSVDataCloudWriter(BaseDataCloudWriter):
             name = f"{name}{SUFFIX}"
         dataframe.write.csv(name, mode=write_mode)
 
+    def auto_write_to_dlo(self, name: str, dataframe: PySparkDataFrame) -> None:
+        # use overwrite since this is a local only writer
+        self.write_to_dlo(name, dataframe, WriteMode.OVERWRITE)
+
     def write_to_dmo(
         self, name: str, dataframe: PySparkDataFrame, write_mode: WriteMode
     ) -> None:
@@ -43,3 +47,7 @@ class CSVDataCloudWriter(BaseDataCloudWriter):
         if not name.lower().endswith(SUFFIX):
             name = f"{name}{SUFFIX}"
         dataframe.write.csv(name, mode=write_mode)
+
+    def auto_write_to_dmo(self, name: str, dataframe: PySparkDataFrame) -> None:
+        # use overwrite since this is a local only writer
+        self.write_to_dmo(name, dataframe, WriteMode.OVERWRITE)

datacustomcode/io/writer/print.py

@@ -122,6 +122,10 @@ class PrintDataCloudWriter(BaseDataCloudWriter):
 
         dataframe.show()
 
+    def auto_write_to_dlo(self, name: str, dataframe: PySparkDataFrame) -> None:
+        self.validate_dataframe_columns_against_dlo(dataframe, name)
+        dataframe.show()
+
     def write_to_dmo(
         self, name: str, dataframe: PySparkDataFrame, write_mode: WriteMode
     ) -> None:
@@ -130,3 +134,6 @@ class PrintDataCloudWriter(BaseDataCloudWriter):
         # so just show the dataframe.
 
         dataframe.show()
+
+    def auto_write_to_dmo(self, name: str, dataframe: PySparkDataFrame) -> None:
+        dataframe.show()

datacustomcode/run.py

@@ -27,7 +27,6 @@ from typing import (
 from datacustomcode.config import config
 from datacustomcode.einstein_predictions_config import einstein_predictions_config
 from datacustomcode.llm_gateway_config import llm_gateway_config
-from datacustomcode.named_credential_config import named_credential_config
 from datacustomcode.scan import find_base_directory, get_package_type
 
 
@@ -65,9 +64,6 @@ def _update_config_options(profile: Optional[str], sf_cli_org: Optional[str]):
         _set_config_option(
             llm_gateway_config.llm_gateway_config, config_key, sf_cli_org
         )
-        _set_config_option(
-            named_credential_config.named_credential_config, config_key, sf_cli_org
-        )
     elif profile != "default":
         config_key = "credentials_profile"
         _set_config_option(config.reader_config, config_key, profile)
@@ -76,9 +72,6 @@ def _update_config_options(profile: Optional[str], sf_cli_org: Optional[str]):
             einstein_predictions_config.einstein_predictions_config, config_key, profile
         )
         _set_config_option(llm_gateway_config.llm_gateway_config, config_key, profile)
-        _set_config_option(
-            named_credential_config.named_credential_config, config_key, profile
-        )
 
 
 def run_entrypoint(

datacustomcode/templates/script/examples/streaming_deltas/entrypoint.py

@@ -3,46 +3,59 @@
 This example is the streaming counterpart to a normal batch entrypoint. Instead
 of a batch ``Client`` with ``read_dlo`` / ``write_to_dlo`` (which read and write
 a bounded snapshot), it uses a :class:`StreamingClient` and its streaming delta
-methods:
+methods.
 
-* ``client.read_dlo_deltas()`` returns a *streaming* DataFrame over the
-  Change Data Feed of the source DLO. Each row carries the source columns plus
-  change-feed metadata columns (``_record_type``, ``_commit_*``).
-* ``client.write_dlo_deltas(name, df)`` starts a streaming query that writes
-  each micro-batch to the target DLO and returns the ``StreamingQuery`` handle.
-  The runtime owns the trigger, and checkpoint location — the caller only
-  chooses the table.
+The first run of a streaming job will use the run mode INITIAL_SYNC which behaves
+like a batch run on the streaming source. A streaming transform can also use run
+mode REBUILD to do the same thing on demand. Note that these will process all
+source rows and overwrite the target.
 
 The transform in between is ordinary PySpark. Because the source is a change
 feed, keep the metadata columns on the DataFrame you hand to
 ``write_dlo_deltas`` — the sink relies on them to merge changes correctly.
 
-This entrypoint only runs inside the Data Cloud streaming (``DELTA_SYNC``)
-runtime; the local ``datacustomcode run`` readers/writers raise
+This entrypoint only runs inside the Data Cloud runtime;
+ the local ``datacustomcode run`` readers/writers raise
 ``NotImplementedError`` for the delta methods.
 """
 
+from pyspark.sql import DataFrame
 from pyspark.sql.functions import col, upper
 
-from datacustomcode.client import StreamingClient
+from datacustomcode.client import (
+    RunMode,
+    StreamingClient,
+    get_run_mode,
+)
 
 
 def main():
+    target_dlo = "Account_std_copy__dll"
     client = StreamingClient()
 
-    # Streaming DataFrame over the source DLO's change feed.
-    deltas = client.read_dlo_deltas()
-
-    # Ordinary PySpark transform.
-    transformed = deltas.withColumn("description__c", upper(col("description__c")))
-
-    # Start the streaming write. write_dlo_deltas returns the StreamingQuery;
-    # the trigger and checkpoint location are provided by the runtime.
-    query = client.write_dlo_deltas("Account_std_copy__dll", transformed)
-
-    # Drive the query's lifecycle. In the streaming runtime this blocks until
-    # the job is stopped by the platform.
-    query.awaitTermination()
+    if get_run_mode() == RunMode.DELTA_SYNC:
+        # Streaming DataFrame over the source DLO's change feed.
+        dataframe = client.read_dlo_deltas()
+        # Ordinary PySpark transform.
+        transformed = transform(dataframe)
+
+        # Start the streaming write. write_dlo_deltas returns the StreamingQuery;
+        # the trigger and checkpoint location are provided by the runtime.
+        query = client.write_dlo_deltas(target_dlo, transformed)
+
+        # Drive the query's lifecycle. In the streaming runtime this blocks until
+        # the job is stopped by the platform.
+        query.awaitTermination()
+    else:
+        # initial sync and rebuild read the entire streaming source DLO and
+        # write using a server-decided mode based on the run mode
+        dataframe = client.read_dlo()
+        transformed = transform(dataframe)
+        client.auto_write_to_dlo(target_dlo, transformed)
+
+
+def transform(dataframe: DataFrame) -> DataFrame:
+    return dataframe.withColumn("description__c", upper(col("description__c")))
 
 
 if __name__ == "__main__":

datacustomcode/templates/script/jupyterlab.sh

@@ -45,13 +45,24 @@ check_docker() {
     echo "Docker daemon is running"
 }
 
+# Function to check if openssl is installed
+check_openssl() {
+    if ! command -v openssl &> /dev/null; then
+        echo "openssl is not installed. It is required to generate a secure JupyterLab access token."
+        exit 1
+    fi
+}
+
 # Function to start Jupyter server
 start_jupyter() {
     echo "Building the docker image"
     docker build -t datacloud-customcode .
 
+    local TOKEN
+    TOKEN=$(openssl rand -hex 32)
+
     echo "Running the docker container"
-    docker run -d --rm -p 8888:8888 \
+    docker run -d --rm -p 127.0.0.1:8888:8888 \
         -v $(pwd):/workspace \
         --name jupyter-server \
         datacloud-customcode jupyter lab \
@@ -59,12 +70,14 @@ start_jupyter() {
         --port=8888 \
         --no-browser \
         --allow-root \
-        --NotebookApp.token='' \
-        --NotebookApp.password='' \
+        --NotebookApp.token="$TOKEN" \
         --notebook-dir=/workspace
 
     sleep 3  # Wait for server to start
-    open_browser "http://localhost:8888"
+    local URL
+    URL="http://localhost:8888/?token=$TOKEN"
+    echo "Opening $URL"
+    open_browser $URL
 }
 
 # Function to stop Jupyter server
@@ -82,6 +95,7 @@ stop_jupyter() {
 case "$1" in
     "start")
         check_docker
+        check_openssl
         start_jupyter
         ;;
     "stop")

{salesforce_data_customcode-6.1.0.dev4.dist-info → salesforce_data_customcode-6.1.0.dev5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: salesforce-data-customcode
-Version: 6.1.0.dev4
+Version: 6.1.0.dev5
 Summary: Data Cloud Custom Code SDK
 License-Expression: Apache-2.0
 License-File: LICENSE.txt
@@ -534,7 +534,7 @@ exploration.  Instead of running an entire script, one can run one code cell at
 
 You can read more about Jupyter Notebooks here: https://jupyter.org/
 
-1. Within the root project of your package folder, run `./jupyterlab.sh start`
+1. Within the root project of your package folder, run `./jupyterlab.sh start`. This prints an access token and opens an already-authenticated JupyterLab session in your browser. If the browser doesn't open automatically, copy the printed `http://localhost:8888/?token=...` URL into your browser.
 1. Double-click on "account.ipynb" file, which provides a starting point for a notebook
 1. Use shift+enter to execute each cell within the notebook.  Add/edit/delete cells of code as needed for your data exploration.
 1. Don't forget to run `./jupyterlab.sh stop` to stop the docker container
@@ -625,5 +625,5 @@ If you're using OAuth Tokens authentication, the initial configure will retrieve
 ## Other docs
 
 - [Troubleshooting](./docs/troubleshooting.md)
-- [For Contributors](./FOR_CONTRIBUTORS.md)
+- [Contributing](./CONTRIBUTING.md)