salesforce-data-customcode 6.1.0.dev1__tar.gz → 6.1.0.dev2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/PKG-INFO +3 -40
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/README.md +2 -39
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/pyproject.toml +1 -1
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/__init__.py +0 -5
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/cli.py +4 -29
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/client.py +192 -211
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/config.py +0 -5
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/config.yaml +6 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/constants.py +0 -8
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/deploy.py +23 -57
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/function/runtime.py +16 -0
- salesforce_data_customcode-6.1.0.dev2/src/datacustomcode/io/reader/base.py +43 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/io/writer/base.py +0 -33
- salesforce_data_customcode-6.1.0.dev2/src/datacustomcode/named_credential/__init__.py +26 -0
- salesforce_data_customcode-6.1.0.dev2/src/datacustomcode/named_credential/base.py +54 -0
- salesforce_data_customcode-6.1.0.dev2/src/datacustomcode/named_credential/default.py +93 -0
- salesforce_data_customcode-6.1.0.dev2/src/datacustomcode/named_credential/direct/__init__.py +19 -0
- salesforce_data_customcode-6.1.0.dev2/src/datacustomcode/named_credential/direct/auth.py +63 -0
- salesforce_data_customcode-6.1.0.dev2/src/datacustomcode/named_credential/direct/credentials.py +121 -0
- salesforce_data_customcode-6.1.0.dev2/src/datacustomcode/named_credential/direct/transport.py +110 -0
- salesforce_data_customcode-6.1.0.dev2/src/datacustomcode/named_credential/direct/url_resolver.py +112 -0
- salesforce_data_customcode-6.1.0.dev2/src/datacustomcode/named_credential/spark_base.py +93 -0
- salesforce_data_customcode-6.1.0.dev2/src/datacustomcode/named_credential/spark_default.py +154 -0
- salesforce_data_customcode-6.1.0.dev2/src/datacustomcode/named_credential/types/__init__.py +14 -0
- salesforce_data_customcode-6.1.0.dev2/src/datacustomcode/named_credential/types/http_method.py +29 -0
- salesforce_data_customcode-6.1.0.dev2/src/datacustomcode/named_credential/types/http_request.py +63 -0
- salesforce_data_customcode-6.1.0.dev2/src/datacustomcode/named_credential/types/http_request_builder.py +55 -0
- salesforce_data_customcode-6.1.0.dev2/src/datacustomcode/named_credential/types/http_response.py +43 -0
- salesforce_data_customcode-6.1.0.dev2/src/datacustomcode/named_credential/types/http_response_builder.py +24 -0
- salesforce_data_customcode-6.1.0.dev2/src/datacustomcode/named_credential_config.py +105 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/run.py +7 -11
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/scan.py +29 -164
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/template.py +1 -13
- salesforce_data_customcode-6.1.0.dev2/src/datacustomcode/templates/function/example/chunking_with_external_callout/README.md +119 -0
- salesforce_data_customcode-6.1.0.dev2/src/datacustomcode/templates/function/example/chunking_with_external_callout/entrypoint.py +161 -0
- salesforce_data_customcode-6.1.0.dev2/src/datacustomcode/templates/function/example/chunking_with_external_callout/external_callout_config.json +11 -0
- salesforce_data_customcode-6.1.0.dev2/src/datacustomcode/templates/function/example/chunking_with_external_callout/tests/test.json +16 -0
- salesforce_data_customcode-6.1.0.dev2/src/datacustomcode/templates/function/example/chunking_with_prediction/config.json +3 -0
- salesforce_data_customcode-6.1.0.dev1/src/datacustomcode/io/reader/base.py +0 -85
- salesforce_data_customcode-6.1.0.dev1/src/datacustomcode/templates/script/examples/streaming_deltas/entrypoint.py +0 -49
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/LICENSE.txt +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/auth.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/cmd.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/common_config.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/credentials.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/einstein_platform_client.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/einstein_platform_config.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/einstein_predictions/__init__.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/einstein_predictions/base.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/einstein_predictions/errors.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/einstein_predictions/impl/default.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/einstein_predictions/spark_base.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/einstein_predictions/spark_default.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/einstein_predictions/types.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/einstein_predictions_config.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/file/__init__.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/file/base.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/file/path/__init__.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/file/path/default.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/function/__init__.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/function/base.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/function/feature_types/__init__.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/function/feature_types/chunking.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/function_utils.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/io/__init__.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/io/base.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/io/reader/__init__.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/io/reader/query_api.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/io/reader/sf_cli.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/io/reader/utils.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/io/writer/__init__.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/io/writer/csv.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/io/writer/print.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/llm_gateway/__init__.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/llm_gateway/base.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/llm_gateway/default.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/llm_gateway/errors.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/llm_gateway/spark_base.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/llm_gateway/spark_default.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/llm_gateway/types/__init__.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/llm_gateway/types/generate_text_request.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/llm_gateway/types/generate_text_request_builder.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/llm_gateway/types/generate_text_response.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/llm_gateway/types/generate_text_response_builder.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/llm_gateway_config.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/mixin.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/py.typed +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/spark/__init__.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/spark/base.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/spark/default.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/templates/__init__.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/templates/function/.devcontainer/devcontainer.json +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/templates/function/Dockerfile.dependencies +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/templates/function/README.md +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/templates/function/build_native_dependencies.sh +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/templates/function/chunking/payload/config.json +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/templates/function/chunking/payload/entrypoint.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/templates/function/chunking/requirements.txt +0 -0
- {salesforce_data_customcode-6.1.0.dev1/src/datacustomcode/templates/function/example/chunking_with_llm → salesforce_data_customcode-6.1.0.dev2/src/datacustomcode/templates/function/example/chunking_with_external_callout}/config.json +0 -0
- {salesforce_data_customcode-6.1.0.dev1/src/datacustomcode/templates/function/example/chunking_with_prediction → salesforce_data_customcode-6.1.0.dev2/src/datacustomcode/templates/function/example/chunking_with_llm}/config.json +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/templates/function/example/chunking_with_llm/entrypoint.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/templates/function/example/chunking_with_llm/files/chunking_prompt.txt +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/templates/function/example/chunking_with_llm/tests/test.json +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/templates/function/example/chunking_with_prediction/entrypoint.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/templates/function/example/chunking_with_prediction/tests/test.json +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/templates/function/payload/config.json +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/templates/function/payload/entrypoint.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/templates/function/payload/utility.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/templates/function/requirements-dev.txt +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/templates/function/requirements.txt +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/templates/script/.devcontainer/devcontainer.json +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/templates/script/Dockerfile +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/templates/script/Dockerfile.dependencies +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/templates/script/README.md +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/templates/script/account.ipynb +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/templates/script/build_native_dependencies.sh +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/templates/script/examples/employee_hierarchy/employee_data.csv +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/templates/script/examples/employee_hierarchy/entrypoint.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/templates/script/jupyterlab.sh +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/templates/script/payload/config.json +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/templates/script/payload/entrypoint.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/templates/script/requirements-dev.txt +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/templates/script/requirements.txt +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/token_provider.py +0 -0
- {salesforce_data_customcode-6.1.0.dev1 → salesforce_data_customcode-6.1.0.dev2}/src/datacustomcode/version.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: salesforce-data-customcode
|
|
3
|
-
Version: 6.1.0.
|
|
3
|
+
Version: 6.1.0.dev2
|
|
4
4
|
Summary: Data Cloud Custom Code SDK
|
|
5
5
|
License-Expression: Apache-2.0
|
|
6
6
|
License-File: LICENSE.txt
|
|
@@ -170,22 +170,15 @@ Your Python dependencies can be packaged as .py files, .zip archives (containing
|
|
|
170
170
|
|
|
171
171
|
## API
|
|
172
172
|
|
|
173
|
-
Your entry point script will define logic using the `Client` object
|
|
173
|
+
Your entry point script will define logic using the `Client` object which wraps data access layers.
|
|
174
174
|
|
|
175
|
-
|
|
175
|
+
You should only need the following methods:
|
|
176
176
|
* `find_file_path(file_name)` – Resolve a bundled file (placed under `payload/files/`) to a `pathlib.Path` that exists. Works the same locally and inside Data Cloud — see [Bundled file resolution](#bundled-file-resolution) below for the full lookup order. Raises `FileNotFoundError` if the file isn't found.
|
|
177
177
|
* `read_dlo(name)` – Read from a Data Lake Object by name
|
|
178
178
|
* `read_dmo(name)` – Read from a Data Model Object by name
|
|
179
179
|
* `write_to_dlo(name, spark_dataframe, write_mode)` – Write to a Data Model Object by name with a Spark dataframe
|
|
180
180
|
* `write_to_dmo(name, spark_dataframe, write_mode)` – Write to a Data Lake Object by name with a Spark dataframe
|
|
181
181
|
|
|
182
|
-
For a streaming (delta) transform, use `StreamingClient`, which exposes the streaming counterparts:
|
|
183
|
-
* `read_dlo_deltas()` – Read the streaming change feed (deltas) of a Data Lake Object as a streaming DataFrame.
|
|
184
|
-
* `read_dmo_deltas()` – Read the streaming change feed (deltas) of a Data Model Object as a streaming DataFrame.
|
|
185
|
-
* `write_dlo_deltas(name, spark_dataframe)` – Write a streaming DataFrame of deltas to a Data Lake Object; returns the started `StreamingQuery`
|
|
186
|
-
|
|
187
|
-
`find_file_path`, `llm_gateway_generate_text`, and `einstein_predict` are available on both clients.
|
|
188
|
-
|
|
189
182
|
For example:
|
|
190
183
|
```python
|
|
191
184
|
from datacustomcode import Client
|
|
@@ -201,36 +194,6 @@ client.write_to_dlo('output_DLO')
|
|
|
201
194
|
> [!WARNING]
|
|
202
195
|
> Currently we only support reading from DMOs and writing to DMOs or reading from DLOs and writing to DLOs, but they cannot mix.
|
|
203
196
|
|
|
204
|
-
### Streaming (delta) transforms
|
|
205
|
-
|
|
206
|
-
Streaming BYOC transforms process a Data Lake Object's Change Data Feed continuously instead of reading a bounded snapshot. Use a `StreamingClient` and its `*_deltas` methods in place of the batch `Client` read/write methods:
|
|
207
|
-
|
|
208
|
-
```python
|
|
209
|
-
from pyspark.sql.functions import col, upper
|
|
210
|
-
|
|
211
|
-
from datacustomcode import StreamingClient
|
|
212
|
-
|
|
213
|
-
client = StreamingClient()
|
|
214
|
-
|
|
215
|
-
# read_dlo_deltas returns a *streaming* DataFrame over the change feed.
|
|
216
|
-
# The runtime resolves the single streaming source, so no name is passed.
|
|
217
|
-
deltas = client.read_dlo_deltas()
|
|
218
|
-
|
|
219
|
-
# Ordinary PySpark transform.
|
|
220
|
-
transformed = deltas.withColumn("description__c", upper(col("description__c")))
|
|
221
|
-
|
|
222
|
-
# write_dlo_deltas starts a streaming query and returns the StreamingQuery.
|
|
223
|
-
# The runtime owns the trigger and checkpoint location; you
|
|
224
|
-
# choose only the target table.
|
|
225
|
-
query = client.write_dlo_deltas("Output__dll", transformed)
|
|
226
|
-
query.awaitTermination()
|
|
227
|
-
```
|
|
228
|
-
|
|
229
|
-
Notes:
|
|
230
|
-
|
|
231
|
-
- These methods only run inside the Data Cloud streaming (`DELTA_SYNC`) runtime. Locally (`datacustomcode run`) they raise `NotImplementedError`, since there is no change feed to stream.
|
|
232
|
-
- A complete runnable entry point is provided in [`examples/streaming_deltas/entrypoint.py`](src/datacustomcode/templates/script/examples/streaming_deltas/entrypoint.py).
|
|
233
|
-
|
|
234
197
|
### Bundled file resolution
|
|
235
198
|
|
|
236
199
|
Place bundled files (CSVs, prompt files, etc.) under `payload/files/`. The same `client.find_file_path("data.csv")` call resolves consistently across all three runtimes:
|
|
@@ -145,22 +145,15 @@ Your Python dependencies can be packaged as .py files, .zip archives (containing
|
|
|
145
145
|
|
|
146
146
|
## API
|
|
147
147
|
|
|
148
|
-
Your entry point script will define logic using the `Client` object
|
|
148
|
+
Your entry point script will define logic using the `Client` object which wraps data access layers.
|
|
149
149
|
|
|
150
|
-
|
|
150
|
+
You should only need the following methods:
|
|
151
151
|
* `find_file_path(file_name)` – Resolve a bundled file (placed under `payload/files/`) to a `pathlib.Path` that exists. Works the same locally and inside Data Cloud — see [Bundled file resolution](#bundled-file-resolution) below for the full lookup order. Raises `FileNotFoundError` if the file isn't found.
|
|
152
152
|
* `read_dlo(name)` – Read from a Data Lake Object by name
|
|
153
153
|
* `read_dmo(name)` – Read from a Data Model Object by name
|
|
154
154
|
* `write_to_dlo(name, spark_dataframe, write_mode)` – Write to a Data Model Object by name with a Spark dataframe
|
|
155
155
|
* `write_to_dmo(name, spark_dataframe, write_mode)` – Write to a Data Lake Object by name with a Spark dataframe
|
|
156
156
|
|
|
157
|
-
For a streaming (delta) transform, use `StreamingClient`, which exposes the streaming counterparts:
|
|
158
|
-
* `read_dlo_deltas()` – Read the streaming change feed (deltas) of a Data Lake Object as a streaming DataFrame.
|
|
159
|
-
* `read_dmo_deltas()` – Read the streaming change feed (deltas) of a Data Model Object as a streaming DataFrame.
|
|
160
|
-
* `write_dlo_deltas(name, spark_dataframe)` – Write a streaming DataFrame of deltas to a Data Lake Object; returns the started `StreamingQuery`
|
|
161
|
-
|
|
162
|
-
`find_file_path`, `llm_gateway_generate_text`, and `einstein_predict` are available on both clients.
|
|
163
|
-
|
|
164
157
|
For example:
|
|
165
158
|
```python
|
|
166
159
|
from datacustomcode import Client
|
|
@@ -176,36 +169,6 @@ client.write_to_dlo('output_DLO')
|
|
|
176
169
|
> [!WARNING]
|
|
177
170
|
> Currently we only support reading from DMOs and writing to DMOs or reading from DLOs and writing to DLOs, but they cannot mix.
|
|
178
171
|
|
|
179
|
-
### Streaming (delta) transforms
|
|
180
|
-
|
|
181
|
-
Streaming BYOC transforms process a Data Lake Object's Change Data Feed continuously instead of reading a bounded snapshot. Use a `StreamingClient` and its `*_deltas` methods in place of the batch `Client` read/write methods:
|
|
182
|
-
|
|
183
|
-
```python
|
|
184
|
-
from pyspark.sql.functions import col, upper
|
|
185
|
-
|
|
186
|
-
from datacustomcode import StreamingClient
|
|
187
|
-
|
|
188
|
-
client = StreamingClient()
|
|
189
|
-
|
|
190
|
-
# read_dlo_deltas returns a *streaming* DataFrame over the change feed.
|
|
191
|
-
# The runtime resolves the single streaming source, so no name is passed.
|
|
192
|
-
deltas = client.read_dlo_deltas()
|
|
193
|
-
|
|
194
|
-
# Ordinary PySpark transform.
|
|
195
|
-
transformed = deltas.withColumn("description__c", upper(col("description__c")))
|
|
196
|
-
|
|
197
|
-
# write_dlo_deltas starts a streaming query and returns the StreamingQuery.
|
|
198
|
-
# The runtime owns the trigger and checkpoint location; you
|
|
199
|
-
# choose only the target table.
|
|
200
|
-
query = client.write_dlo_deltas("Output__dll", transformed)
|
|
201
|
-
query.awaitTermination()
|
|
202
|
-
```
|
|
203
|
-
|
|
204
|
-
Notes:
|
|
205
|
-
|
|
206
|
-
- These methods only run inside the Data Cloud streaming (`DELTA_SYNC`) runtime. Locally (`datacustomcode run`) they raise `NotImplementedError`, since there is no change feed to stream.
|
|
207
|
-
- A complete runnable entry point is provided in [`examples/streaming_deltas/entrypoint.py`](src/datacustomcode/templates/script/examples/streaming_deltas/entrypoint.py).
|
|
208
|
-
|
|
209
172
|
### Bundled file resolution
|
|
210
173
|
|
|
211
174
|
Place bundled files (CSVs, prompt files, etc.) under `payload/files/`. The same `client.find_file_path("data.csv")` call resolves consistently across all three runtimes:
|
|
@@ -23,7 +23,6 @@ __all__ = [
|
|
|
23
23
|
"QueryAPIDataCloudReader",
|
|
24
24
|
"SparkEinsteinPredictions",
|
|
25
25
|
"SparkLLMGateway",
|
|
26
|
-
"StreamingClient",
|
|
27
26
|
"einstein_predict_col",
|
|
28
27
|
"llm_gateway_generate_text_col",
|
|
29
28
|
]
|
|
@@ -35,10 +34,6 @@ def __getattr__(name: str):
|
|
|
35
34
|
from datacustomcode.client import Client
|
|
36
35
|
|
|
37
36
|
return Client
|
|
38
|
-
elif name == "StreamingClient":
|
|
39
|
-
from datacustomcode.client import StreamingClient
|
|
40
|
-
|
|
41
|
-
return StreamingClient
|
|
42
37
|
elif name == "AuthType":
|
|
43
38
|
from datacustomcode.credentials import AuthType
|
|
44
39
|
|
|
@@ -283,19 +283,10 @@ def deploy(
|
|
|
283
283
|
)
|
|
284
284
|
@click.option(
|
|
285
285
|
"--use-in-feature",
|
|
286
|
-
"
|
|
287
|
-
|
|
288
|
-
help=(
|
|
289
|
-
"Invoke option for this package. For scripts: 'BatchTransform' "
|
|
290
|
-
"(default) or 'StreamingTransform'. For functions: 'SearchIndexChunking'."
|
|
291
|
-
),
|
|
286
|
+
default="SearchIndexChunking",
|
|
287
|
+
help="Feature where this function will be used (only applicable for function).",
|
|
292
288
|
)
|
|
293
289
|
def init(directory: str, code_type: str, use_in_feature: Optional[str]):
|
|
294
|
-
from datacustomcode.constants import (
|
|
295
|
-
SCRIPT_USE_IN_FEATURE_BATCH,
|
|
296
|
-
SCRIPT_USE_IN_FEATURE_OPTIONS,
|
|
297
|
-
SCRIPT_USE_IN_FEATURE_STREAMING,
|
|
298
|
-
)
|
|
299
290
|
from datacustomcode.scan import (
|
|
300
291
|
dc_config_json_from_file,
|
|
301
292
|
update_config,
|
|
@@ -303,23 +294,9 @@ def init(directory: str, code_type: str, use_in_feature: Optional[str]):
|
|
|
303
294
|
)
|
|
304
295
|
from datacustomcode.template import copy_function_template, copy_script_template
|
|
305
296
|
|
|
306
|
-
streaming = False
|
|
307
|
-
if code_type == "script":
|
|
308
|
-
use_in_feature = use_in_feature or SCRIPT_USE_IN_FEATURE_BATCH
|
|
309
|
-
if use_in_feature not in SCRIPT_USE_IN_FEATURE_OPTIONS:
|
|
310
|
-
click.secho(
|
|
311
|
-
f"Error: Invalid --use-in-feature '{use_in_feature}' for a "
|
|
312
|
-
f"script. Valid options: {', '.join(SCRIPT_USE_IN_FEATURE_OPTIONS)}.",
|
|
313
|
-
fg="red",
|
|
314
|
-
)
|
|
315
|
-
raise click.Abort()
|
|
316
|
-
streaming = use_in_feature == SCRIPT_USE_IN_FEATURE_STREAMING
|
|
317
|
-
else:
|
|
318
|
-
use_in_feature = use_in_feature or "SearchIndexChunking"
|
|
319
|
-
|
|
320
297
|
click.echo("Copying template to " + click.style(directory, fg="blue", bold=True))
|
|
321
298
|
if code_type == "script":
|
|
322
|
-
copy_script_template(directory
|
|
299
|
+
copy_script_template(directory)
|
|
323
300
|
elif code_type == "function":
|
|
324
301
|
copy_function_template(directory, use_in_feature)
|
|
325
302
|
entrypoint_path = os.path.join(directory, PAYLOAD_DIR, ENTRYPOINT_FILE)
|
|
@@ -329,9 +306,7 @@ def init(directory: str, code_type: str, use_in_feature: Optional[str]):
|
|
|
329
306
|
sdk_config = {"type": code_type}
|
|
330
307
|
write_sdk_config(directory, sdk_config)
|
|
331
308
|
|
|
332
|
-
config_json = dc_config_json_from_file(
|
|
333
|
-
entrypoint_path, code_type, streaming=streaming
|
|
334
|
-
)
|
|
309
|
+
config_json = dc_config_json_from_file(entrypoint_path, code_type)
|
|
335
310
|
with open(config_location, "w") as f:
|
|
336
311
|
json.dump(config_json, f, indent=2)
|
|
337
312
|
|