salesforce-data-customcode 6.1.0.dev1__py3-none-any.whl → 6.1.0.dev3__py3-none-any.whl

datacustomcode/__init__.py

@@ -23,7 +23,6 @@ __all__ = [
     "QueryAPIDataCloudReader",
     "SparkEinsteinPredictions",
     "SparkLLMGateway",
-    "StreamingClient",
     "einstein_predict_col",
     "llm_gateway_generate_text_col",
 ]
@@ -35,10 +34,6 @@ def __getattr__(name: str):
         from datacustomcode.client import Client
 
         return Client
-    elif name == "StreamingClient":
-        from datacustomcode.client import StreamingClient
-
-        return StreamingClient
     elif name == "AuthType":
         from datacustomcode.credentials import AuthType
 

datacustomcode/cli.py

@@ -262,7 +262,7 @@ def deploy(
         mapped_feature = USE_IN_FEATURE_MAPPING_FOR_CONNECT_API.get(
             use_in_feature, use_in_feature
         )
-        metadata.functionInvokeOptions = [mapped_feature]
+        metadata.invokeOptions = [mapped_feature]
 
     try:
         if sf_cli_org:
@@ -283,19 +283,10 @@ def deploy(
 )
 @click.option(
     "--use-in-feature",
-    "-u",
-    default=None,
-    help=(
-        "Invoke option for this package. For scripts: 'BatchTransform' "
-        "(default) or 'StreamingTransform'. For functions: 'SearchIndexChunking'."
-    ),
+    default="SearchIndexChunking",
+    help="Feature where this function will be used (only applicable for function).",
 )
 def init(directory: str, code_type: str, use_in_feature: Optional[str]):
-    from datacustomcode.constants import (
-        SCRIPT_USE_IN_FEATURE_BATCH,
-        SCRIPT_USE_IN_FEATURE_OPTIONS,
-        SCRIPT_USE_IN_FEATURE_STREAMING,
-    )
     from datacustomcode.scan import (
         dc_config_json_from_file,
         update_config,
@@ -303,23 +294,9 @@ def init(directory: str, code_type: str, use_in_feature: Optional[str]):
     )
     from datacustomcode.template import copy_function_template, copy_script_template
 
-    streaming = False
-    if code_type == "script":
-        use_in_feature = use_in_feature or SCRIPT_USE_IN_FEATURE_BATCH
-        if use_in_feature not in SCRIPT_USE_IN_FEATURE_OPTIONS:
-            click.secho(
-                f"Error: Invalid --use-in-feature '{use_in_feature}' for a "
-                f"script. Valid options: {', '.join(SCRIPT_USE_IN_FEATURE_OPTIONS)}.",
-                fg="red",
-            )
-            raise click.Abort()
-        streaming = use_in_feature == SCRIPT_USE_IN_FEATURE_STREAMING
-    else:
-        use_in_feature = use_in_feature or "SearchIndexChunking"
-
     click.echo("Copying template to " + click.style(directory, fg="blue", bold=True))
     if code_type == "script":
-        copy_script_template(directory, streaming=streaming)
+        copy_script_template(directory)
     elif code_type == "function":
         copy_function_template(directory, use_in_feature)
     entrypoint_path = os.path.join(directory, PAYLOAD_DIR, ENTRYPOINT_FILE)
@@ -329,9 +306,7 @@ def init(directory: str, code_type: str, use_in_feature: Optional[str]):
     sdk_config = {"type": code_type}
     write_sdk_config(directory, sdk_config)
 
-    config_json = dc_config_json_from_file(
-        entrypoint_path, code_type, streaming=streaming
-    )
+    config_json = dc_config_json_from_file(entrypoint_path, code_type)
     with open(config_location, "w") as f:
         json.dump(config_json, f, indent=2)
 

datacustomcode/client.py

@@ -21,9 +21,7 @@ from typing import (
     ClassVar,
     Dict,
     Optional,
-    TypeVar,
     Union,
-    cast,
 )
 
 from datacustomcode.config import config
@@ -31,60 +29,25 @@ from datacustomcode.einstein_predictions_config import spark_einstein_prediction
 from datacustomcode.file.path.default import DefaultFindFilePath
 from datacustomcode.io.reader.base import BaseDataCloudReader
 from datacustomcode.llm_gateway_config import spark_llm_gateway_config
+from datacustomcode.named_credential_config import spark_named_credential_config
 from datacustomcode.spark.default import DefaultSparkSessionProvider
 
 if TYPE_CHECKING:
     from pathlib import Path
 
-    from pyspark.sql import (
-        Column,
-        DataFrame as PySparkDataFrame,
-        SparkSession,
-    )
-    from pyspark.sql.streaming import StreamingQuery
+    from pyspark.sql import Column, DataFrame as PySparkDataFrame
 
     from datacustomcode.einstein_predictions.spark_base import SparkEinsteinPredictions
     from datacustomcode.einstein_predictions.types import PredictionType
     from datacustomcode.io.reader.base import BaseDataCloudReader
     from datacustomcode.io.writer.base import BaseDataCloudWriter, WriteMode
     from datacustomcode.llm_gateway.spark_base import SparkLLMGateway
+    from datacustomcode.named_credential.spark_base import SparkNamedCredential
+    from datacustomcode.named_credential.types.http_request import HTTPRequest
+    from datacustomcode.named_credential.types.http_response import HTTPResponse
     from datacustomcode.spark.base import BaseSparkSessionProvider
 
 
-def _streaming_source_name() -> str:
-    """Return the streaming transform's read-source name.
-
-    Resolved from ``config.streaming_source``, which ``run_entrypoint``
-    populates from config.json's ``streamingSource`` field.
-
-    Raises:
-        RuntimeError: If no ``streaming_source`` has been configured (e.g. the
-            transform's config.json has no ``streamingSource`` field).
-    """
-    source = config.streaming_source
-    if not source:
-        raise RuntimeError(
-            "No streaming source configured. A streaming transform must declare "
-            "its read source in config.json under 'streamingSource'."
-        )
-    return source
-
-
-def _active_client() -> "_BaseClient":
-    """Return the client backing the module-level Spark column helpers.
-
-    Prefers an already-initialized singleton so a streaming job reuses its
-    :class:`StreamingClient` (and a batch job its :class:`Client`) rather than
-    forcing an unrelated client into existence. Falls back to building the
-    batch :class:`Client` when neither has been created yet.
-    """
-    if Client._instance is not None:
-        return Client._instance
-    if StreamingClient._instance is not None:
-        return StreamingClient._instance
-    return Client()
-
-
 def _build_spark_llm_gateway() -> "SparkLLMGateway":
     """Instantiate the SDK-configured :class:`SparkLLMGateway`.
 
@@ -140,7 +103,7 @@ def llm_gateway_generate_text_col(
         the generated text; on failure, ``status == "ERROR"`` and the
         ``error_*`` fields carry diagnostic detail.
     """
-    gateway = _active_client()._get_spark_llm_gateway()
+    gateway = Client()._get_spark_llm_gateway()
     return gateway.llm_gateway_generate_text_col(template, values, model_id=model_id)
 
 
@@ -159,6 +122,21 @@ def _build_spark_einstein_predictions() -> "SparkEinsteinPredictions":
     return cfg.to_object()
 
 
+def _build_spark_named_credential() -> "SparkNamedCredential":
+    """Instantiate the SDK-configured :class:`SparkNamedCredential`.
+
+    Raises:
+        RuntimeError: If no ``spark_named_credential_config`` has been loaded.
+    """
+    cfg = spark_named_credential_config.spark_named_credential_config
+    if cfg is None:
+        raise RuntimeError(
+            "spark_named_credential_config is not configured. Add a "
+            "'spark_named_credential_config' section to config.yaml."
+        )
+    return cfg.to_object()
+
+
 def einstein_predict_col(
     model_api_name: str,
     prediction_type: "PredictionType",
@@ -202,12 +180,46 @@ def einstein_predict_col(
         the JSON-serialized prediction payload; on failure, ``status ==
         "ERROR"`` and the ``error_*`` fields carry diagnostic detail.
     """
-    predictions = _active_client()._get_spark_einstein_predictions()
+    predictions = Client()._get_spark_einstein_predictions()
     return predictions.einstein_predict_col(
         model_api_name, prediction_type, features, settings=settings
     )
 
 
+def named_credential_request_col(
+    request: "HTTPRequest",
+    body: Optional["Column"] = None,
+) -> "Column":
+    """Build a Spark Column that makes one Named Credential callout per row.
+
+    The endpoint, method, and headers are fixed for the call (taken from
+    ``request``); only ``body`` varies per row. Use this instead of
+    :meth:`Client.named_credential_request` when the callout runs across a
+    DataFrame so each row is dispatched independently rather than one-shot on
+    the driver.
+
+    The returned Column yields a struct ``{status, response, error_code,
+    error_message}`` for each row. ``response`` is itself a struct
+    ``{status_code, body, headers}``. Use ``[...]`` to pick a field, e.g.
+    ``named_credential_request_col(...)["response"]["status_code"]``. A transport
+    failure sets ``status`` to ``ERROR`` and populates ``error_message`` (a non-2xx
+    HTTP response is still ``SUCCESS`` with its code in ``response.status_code``),
+    so a single bad row does not abort the whole Spark job.
+
+    Args:
+        request: The callout template — its symbolic reference, method, and
+            headers are applied to every row.
+        body: Optional per-row ``Column`` holding the request body as a
+            string (or null for no body).
+
+    Returns:
+        A Spark ``Column`` of ``StructType`` with fields ``status``,
+        ``response``, ``error_code``, and ``error_message``.
+    """
+    named_credential = Client()._get_spark_named_credential()
+    return named_credential.request_col(request, body=body)
+
+
 class DataCloudObjectType(Enum):
     DLO = "dlo"
     DMO = "dmo"
@@ -246,86 +258,83 @@ class DataCloudAccessLayerException(Exception):
         return msg
 
 
-_ClientT = TypeVar("_ClientT", bound="_BaseClient")
+class Client:
+    """Entrypoint for accessing DataCloud objects.
 
+    This is the object used to access Data Cloud DLOs and DMOs. Accessing DLOs/DMOs
+    are tracked and will throw an exception if they are mixed. In other words, you
+    can read from DLOs and write to DLOs, read from DMOs and write to DMOs, but you
+    cannot read from DLOs and write to DMOs or read from DMOs and write to DLOs.
+    Furthermore you cannot mix during merging tables. This class is a singleton to
+    prevent accidental mixing of DLOs and DMOs.
 
-class _BaseClient:
-    """Shared machinery for the Data Cloud client singletons.
-
-    Holds the wiring common to :class:`Client` (batch) and
-    :class:`StreamingClient`
-
-    This base class is not meant to be instantiated directly; use
-    :class:`Client` or :class:`StreamingClient`.
+    You can provide custom readers and writers to the client for advanced use
+    cases, but this is not recommended for testing as they may result in unexpected
+    behavior once deployed to Data Cloud. By default, the client intercepts all
+    read/write operations and mocks access to Data Cloud. For example, during
+    writing, we print to the console instead of writing to Data Cloud.
 
     Args:
+        finder: Find a file path
         reader: A custom reader to use for reading Data Cloud objects.
         writer: A custom writer to use for writing Data Cloud objects.
-        spark_provider: Optional custom :class:`BaseSparkSessionProvider`.
         spark_llm_gateway: Optional custom :class:`SparkLLMGateway`.
         spark_einstein_predictions: Optional custom
             :class:`SparkEinsteinPredictions`.
+        spark_named_credential: Optional custom :class:`SparkNamedCredential`.
+
+    Example:
+    >>> client = Client()
+    >>> file_path = client.find_file_path("data.csv")
+    >>> dlo = client.read_dlo("my_dlo")
+    >>> client.write_to_dmo("my_dmo", dlo)
+    >>> answer = client.llm_gateway_generate_text("Generate a greeting message")
     """
 
-    # Each concrete subclass gets its own ``_instance`` slot: reads fall through
-    # to this base default of ``None``, but ``cls._instance = ...`` in __new__
-    # always writes to the subclass, so ``Client`` and ``StreamingClient`` never
-    # share an instance.
-    _instance: ClassVar[Optional[_BaseClient]] = None
-    # Process-wide Spark session shared across BOTH client types. Unlike
-    # ``_instance``, this is written via ``_BaseClient._shared_spark`` (never
-    # ``cls._shared_spark``), so the slot lives on the base class and a
-    # ``Client`` and a ``StreamingClient`` in the same process reuse one session
-    # — and therefore one underlying connection — instead of opening two
-    # containing differing state
-    _shared_spark: ClassVar[Optional[SparkSession]] = None
+    _instance: ClassVar[Optional[Client]] = None
     _reader: BaseDataCloudReader
     _writer: BaseDataCloudWriter
     _file: DefaultFindFilePath
     _spark_llm_gateway: Optional[SparkLLMGateway]
     _spark_einstein_predictions: Optional[SparkEinsteinPredictions]
+    _spark_named_credential: Optional[SparkNamedCredential]
     _data_layer_history: dict[DataCloudObjectType, set[str]]
     _code_type: str
 
     def __new__(
-        cls: type[_ClientT],
+        cls,
         reader: Optional[BaseDataCloudReader] = None,
         writer: Optional[BaseDataCloudWriter] = None,
         spark_provider: Optional[BaseSparkSessionProvider] = None,
         spark_llm_gateway: Optional[SparkLLMGateway] = None,
         spark_einstein_predictions: Optional[SparkEinsteinPredictions] = None,
+        spark_named_credential: Optional[SparkNamedCredential] = None,
         code_type: str = "script",
-    ) -> _ClientT:
+    ) -> Client:
 
         if cls._instance is None:
-            instance = super().__new__(cls)
-            instance._spark_llm_gateway = spark_llm_gateway
-            instance._spark_einstein_predictions = spark_einstein_predictions
+            cls._instance = super().__new__(cls)
+            cls._instance._spark_llm_gateway = spark_llm_gateway
+            cls._instance._spark_einstein_predictions = spark_einstein_predictions
+            cls._instance._spark_named_credential = spark_named_credential
             # Initialize Readers and Writers from config
             # and/or provided reader and writer
             if reader is None or writer is None:
-                # We need a spark because we will initialize readers and writers.
-                # Reuse the process-wide session if one client already built it,
-                # so a Client and a StreamingClient share a single connection.
-                if _BaseClient._shared_spark is not None:
-                    spark = _BaseClient._shared_spark
+                # We need a spark because we will initialize readers and writers
+                if config.spark_config is None:
+                    raise ValueError(
+                        "Spark config is required when reader/writer is not provided"
+                    )
+
+                provider: BaseSparkSessionProvider
+                if spark_provider is not None:
+                    provider = spark_provider
+                elif config.spark_provider_config is not None:
+                    provider = config.spark_provider_config.to_object()
                 else:
-                    if config.spark_config is None:
-                        raise ValueError(
-                            "Spark config is required when reader/writer is not "
-                            "provided"
-                        )
-
-                    provider: BaseSparkSessionProvider
-                    if spark_provider is not None:
-                        provider = spark_provider
-                    elif config.spark_provider_config is not None:
-                        provider = config.spark_provider_config.to_object()
-                    else:
-                        provider = DefaultSparkSessionProvider()
-
-                    spark = provider.get_session(config.spark_config)
-                    _BaseClient._shared_spark = spark
+                    provider = DefaultSparkSessionProvider()
+
+                spark = provider.get_session(config.spark_config)
 
             if config.reader_config is None and reader is None:
                 raise ValueError(
@@ -348,17 +357,66 @@ class _BaseClient:
             else:
                 writer_init = writer
 
-            instance._reader = reader_init
-            instance._writer = writer_init
-            instance._file = DefaultFindFilePath()
-            instance._data_layer_history = {
+            cls._instance._reader = reader_init
+            cls._instance._writer = writer_init
+            cls._instance._file = DefaultFindFilePath()
+            cls._instance._data_layer_history = {
                 DataCloudObjectType.DLO: set(),
                 DataCloudObjectType.DMO: set(),
             }
-            cls._instance = instance
-        elif reader is not None or writer is not None:
+        elif (reader is not None or writer is not None) and cls._instance is not None:
             raise ValueError("Cannot set reader or writer after client is initialized")
-        return cast(_ClientT, cls._instance)
+        return cls._instance
+
+    def read_dlo(self, name: str) -> PySparkDataFrame:
+        """Read a DLO from Data Cloud.
+
+        Args:
+            name: The name of the DLO to read.
+
+        Returns:
+            A PySpark DataFrame containing the DLO data.
+        """
+        self._record_dlo_access(name)
+        return self._reader.read_dlo(name)  # type: ignore[no-any-return]
+
+    def read_dmo(self, name: str) -> PySparkDataFrame:
+        """Read a DMO from Data Cloud.
+
+        Args:
+            name: The name of the DMO to read.
+
+        Returns:
+            A PySpark DataFrame containing the DMO data.
+        """
+        self._record_dmo_access(name)
+        return self._reader.read_dmo(name)  # type: ignore[no-any-return]
+
+    def write_to_dlo(
+        self, name: str, dataframe: PySparkDataFrame, write_mode: WriteMode, **kwargs
+    ) -> None:
+        """Write a PySpark DataFrame to a DLO in Data Cloud.
+
+        Args:
+            name: The name of the DLO to write to.
+            dataframe: The PySpark DataFrame to write.
+            write_mode: The write mode to use for writing to the DLO.
+        """
+        self._validate_data_layer_history_does_not_contain(DataCloudObjectType.DMO)
+        return self._writer.write_to_dlo(name, dataframe, write_mode, **kwargs)  # type: ignore[no-any-return]
+
+    def write_to_dmo(
+        self, name: str, dataframe: PySparkDataFrame, write_mode: WriteMode, **kwargs
+    ) -> None:
+        """Write a PySpark DataFrame to a DMO in Data Cloud.
+
+        Args:
+            name: The name of the DMO to write to.
+            dataframe: The PySpark DataFrame to write.
+            write_mode: The write mode to use for writing to the DMO.
+        """
+        self._validate_data_layer_history_does_not_contain(DataCloudObjectType.DLO)
+        return self._writer.write_to_dmo(name, dataframe, write_mode, **kwargs)  # type: ignore[no-any-return]
 
     def find_file_path(self, file_name: str) -> Path:
         """Resolve a bundled file shipped in the package to an absolute path.
@@ -473,6 +531,41 @@ class _BaseClient:
             self._spark_einstein_predictions = _build_spark_einstein_predictions()
         return self._spark_einstein_predictions
 
+    def named_credential_request(
+        self,
+        request: "HTTPRequest",
+        body: Optional[str] = None,
+    ) -> "HTTPResponse":
+        """Issue a one-shot Named Credential external callout. This is the
+        scalar counterpart to :func:`named_credential_request_col`: it runs
+        **once** on the driver — not per row. Use the column helper method
+        instead when you want to fan a callout out across every row of a
+        DataFrame.
+
+        Example:
+
+            >>> from datacustomcode.named_credential.types.http_request_builder \\
+            ...     import HTTPRequestBuilder
+            >>> request = (
+            ...     HTTPRequestBuilder().set_url("callout:NC/search").build()
+            ... )
+            >>> response = Client().named_credential_request(request)
+
+        Args:
+            request: The callout request
+            body: Optional request body. Set the ``Content-Type`` header to
+                match the format; the SDK does not assume or inject one.
+
+        Returns:
+            The external service's response.
+        """
+        return self._get_spark_named_credential().request(request, body=body)
+
+    def _get_spark_named_credential(self) -> SparkNamedCredential:
+        if self._spark_named_credential is None:
+            self._spark_named_credential = _build_spark_named_credential()
+        return self._spark_named_credential
+
     def _validate_data_layer_history_does_not_contain(
         self, data_cloud_object_type: DataCloudObjectType
     ) -> None:
@@ -486,115 +579,3 @@ class _BaseClient:
 
     def _record_dmo_access(self, name: str) -> None:
         self._data_layer_history[DataCloudObjectType.DMO].add(name)
-
-
-class Client(_BaseClient):
-    """Entrypoint for batch access to Data Cloud objects.
-
-    This is the object used to read and write bounded snapshots of Data Cloud
-    DLOs and DMOs.
-    """
-
-    _instance: ClassVar[Optional[Client]] = None
-
-    def read_dlo(self, name: str) -> PySparkDataFrame:
-        """Read a DLO from Data Cloud.
-
-        Args:
-            name: The name of the DLO to read.
-
-        Returns:
-            A PySpark DataFrame containing the DLO data.
-        """
-        self._record_dlo_access(name)
-        return self._reader.read_dlo(name)  # type: ignore[no-any-return]
-
-    def read_dmo(self, name: str) -> PySparkDataFrame:
-        """Read a DMO from Data Cloud.
-
-        Args:
-            name: The name of the DMO to read.
-
-        Returns:
-            A PySpark DataFrame containing the DMO data.
-        """
-        self._record_dmo_access(name)
-        return self._reader.read_dmo(name)  # type: ignore[no-any-return]
-
-    def write_to_dlo(
-        self, name: str, dataframe: PySparkDataFrame, write_mode: WriteMode, **kwargs
-    ) -> None:
-        """Write a PySpark DataFrame to a DLO in Data Cloud.
-
-        Args:
-            name: The name of the DLO to write to.
-            dataframe: The PySpark DataFrame to write.
-            write_mode: The write mode to use for writing to the DLO.
-        """
-        self._validate_data_layer_history_does_not_contain(DataCloudObjectType.DMO)
-        return self._writer.write_to_dlo(name, dataframe, write_mode, **kwargs)  # type: ignore[no-any-return]
-
-    def write_to_dmo(
-        self, name: str, dataframe: PySparkDataFrame, write_mode: WriteMode, **kwargs
-    ) -> None:
-        """Write a PySpark DataFrame to a DMO in Data Cloud.
-
-        Args:
-            name: The name of the DMO to write to.
-            dataframe: The PySpark DataFrame to write.
-            write_mode: The write mode to use for writing to the DMO.
-        """
-        self._validate_data_layer_history_does_not_contain(DataCloudObjectType.DLO)
-        return self._writer.write_to_dmo(name, dataframe, write_mode, **kwargs)  # type: ignore[no-any-return]
-
-
-class StreamingClient(_BaseClient):
-    """Entrypoint for streaming (``DELTA_SYNC``) access to Data Cloud objects.
-
-    This is the streaming counterpart to :class:`Client`. Instead of reading and
-    writing bounded snapshots, it reads a DLO/DMO change feed as a streaming
-    DataFrame and writes the transformed stream back via a ``StreamingQuery``.
-    """
-
-    _instance: ClassVar[Optional[StreamingClient]] = None
-
-    def read_dlo_deltas(self) -> PySparkDataFrame:
-        """Read the streaming change feed (deltas) for a DLO from Data Cloud.
-
-        For use in a streaming (``DELTA_SYNC``) BYOC transform. Returns a
-        streaming DataFrame whose rows carry the change-feed metadata columns
-        (``_record_type``, ``_commit_*``) alongside the source columns.
-
-        Returns:
-            A streaming PySpark DataFrame over the DLO change feed.
-        """
-        self._record_dlo_access(_streaming_source_name())
-        return self._reader.read_dlo_deltas()  # type: ignore[no-any-return]
-
-    def read_dmo_deltas(self) -> PySparkDataFrame:
-        """Read the streaming change feed (deltas) for a DMO from Data Cloud.
-
-        Returns:
-            A streaming PySpark DataFrame over the DMO change feed.
-        """
-        self._record_dmo_access(_streaming_source_name())
-        return self._reader.read_dmo_deltas()  # type: ignore[no-any-return]
-
-    def write_dlo_deltas(
-        self, name: str, dataframe: PySparkDataFrame, **kwargs
-    ) -> StreamingQuery:
-        """Write a streaming DataFrame of deltas to a DLO in Data Cloud.
-
-        Starts a streaming query that writes each micro-batch to the
-        target DLO and returns the  ``StreamingQuery`` handle; the caller
-        typically calls ``query.awaitTermination()``.
-
-        Args:
-            name: The name of the DLO to write to.
-            dataframe: The streaming PySpark DataFrame to write.
-
-        Returns:
-            The started ``StreamingQuery``.
-        """
-        self._validate_data_layer_history_does_not_contain(DataCloudObjectType.DMO)
-        return self._writer.write_dlo_deltas(name, dataframe, **kwargs)  # type: ignore[no-any-return]

datacustomcode/config.py

@@ -89,9 +89,6 @@ class ClientConfig(BaseConfig):
     spark_provider_config: Union[
         SparkProviderConfig[BaseSparkSessionProvider], None
     ] = None
-    # Source object name for a streaming (DELTA_SYNC) transform, populated by
-    # ``run_entrypoint`` from config.json's ``streamingSource`` field
-    streaming_source: Union[str, None] = None
 
     def update(self, other: ClientConfig) -> ClientConfig:
         """Merge this ClientConfig with another, respecting force flags.
@@ -119,8 +116,6 @@ class ClientConfig(BaseConfig):
         self.spark_provider_config = merge(
             self.spark_provider_config, other.spark_provider_config
         )
-        if other.streaming_source is not None:
-            self.streaming_source = other.streaming_source
         return self
 
 

datacustomcode/config.yaml

@@ -34,3 +34,9 @@ llm_gateway_config:
 
 spark_llm_gateway_config:
   type_config_name: DefaultSparkLLMGateway
+
+named_credential_config:
+  type_config_name: DefaultNamedCredential
+
+spark_named_credential_config:
+  type_config_name: DefaultSparkNamedCredential

datacustomcode/constants.py

@@ -35,17 +35,9 @@ FEATURE_TEMPLATE_MAPPING = {
 
 # Feature name to Connect API name mapping
 USE_IN_FEATURE_MAPPING_FOR_CONNECT_API = {
-    "SearchIndexChunking": "UnstructuredChunking",
+    "SearchIndexChunking": "SearchIndexChunking",
 }
 
-# Script (data transform) invoke options
-SCRIPT_USE_IN_FEATURE_BATCH = "BatchTransform"
-SCRIPT_USE_IN_FEATURE_STREAMING = "StreamingTransform"
-SCRIPT_USE_IN_FEATURE_OPTIONS = [
-    SCRIPT_USE_IN_FEATURE_BATCH,
-    SCRIPT_USE_IN_FEATURE_STREAMING,
-]
-
 # Pydantic request/response type names to feature names
 REQUEST_TYPE_TO_FEATURE = {
     "SearchIndexChunkingV1Request": "SearchIndexChunking",