salesforce-data-customcode 0.1.25__tar.gz → 0.1.27__tar.gz

{salesforce_data_customcode-0.1.25 → salesforce_data_customcode-0.1.27}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: salesforce-data-customcode
-Version: 0.1.25
+Version: 0.1.27
 Summary: Data Cloud Custom Code SDK
 License-Expression: Apache-2.0
 License-File: LICENSE.txt
@@ -64,6 +64,8 @@ pip install salesforce-data-customcode
 datacustomcode init my_package
 ```
 
+To create a package of type function, pass the parameter `--code-type=function` with the init command.
+
 This will yield all necessary files to get started:
 ```
 .
@@ -268,6 +270,8 @@ Initialize a new development environment with a code package template.
 
 Argument:
 - `DIRECTORY`: Directory to create project in (default: ".")
+Options:
+- `--code-type TEXT`: This can be either `function` or `script`. The default value is `script` if the argument is missing.
 
 
 #### `datacustomcode scan`
@@ -317,6 +321,7 @@ Options:
 - `--description TEXT`: Description of the transformation job (default: "")
 - `--network TEXT`: docker network (default: "default")
 - `--cpu-size TEXT`: CPU size for the deployment (default: `CPU_2XL`). Available options: CPU_L(Large), CPU_XL(Extra Large), CPU_2XL(2X Large), CPU_4XL(4X Large)
+- `--function-invoke-opt TEXT`: Currently we support only `UnstructuredChunking` for functions.
 
 
 ## Docker usage

{salesforce_data_customcode-0.1.25 → salesforce_data_customcode-0.1.27}/README.md

@@ -40,6 +40,8 @@ pip install salesforce-data-customcode
 datacustomcode init my_package
 ```
 
+To create a package of type function, pass the parameter `--code-type=function` with the init command.
+
 This will yield all necessary files to get started:
 ```
 .
@@ -244,6 +246,8 @@ Initialize a new development environment with a code package template.
 
 Argument:
 - `DIRECTORY`: Directory to create project in (default: ".")
+Options:
+- `--code-type TEXT`: This can be either `function` or `script`. The default value is `script` if the argument is missing.
 
 
 #### `datacustomcode scan`
@@ -293,6 +297,7 @@ Options:
 - `--description TEXT`: Description of the transformation job (default: "")
 - `--network TEXT`: docker network (default: "default")
 - `--cpu-size TEXT`: CPU size for the deployment (default: `CPU_2XL`). Available options: CPU_L(Large), CPU_XL(Extra Large), CPU_2XL(2X Large), CPU_4XL(4X Large)
+- `--function-invoke-opt TEXT`: Currently we support only `UnstructuredChunking` for functions.
 
 
 ## Docker usage

{salesforce_data_customcode-0.1.25 → salesforce_data_customcode-0.1.27}/pyproject.toml

@@ -18,7 +18,7 @@ license = "Apache-2.0"
 name = "salesforce-data-customcode"
 readme = "README.md"
 requires-python = ">=3.10,<3.12"
-version = "0.1.25"
+version = "0.1.27"
 
 [tool.black]
 exclude = '''

{salesforce_data_customcode-0.1.25 → salesforce_data_customcode-0.1.27}/src/datacustomcode/cmd.py

@@ -85,9 +85,13 @@ def _cmd_output(
     **kwargs: Any,
 ) -> tuple[int, bytes, Union[bytes, None]]:
     _setdefault_kwargs(kwargs)
+    kwargs.setdefault("shell", True)
+    # On Windows, Popen with shell=True and a sequence uses list2cmdline() which
+    # quotes the entire string, causing cmd.exe to fail. Joining to a plain string
+    # works correctly on both Unix (/bin/sh -c "...") and Windows (cmd.exe /c ...).
+    cmd_arg: Union[tuple[str, ...], str] = " ".join(cmd) if kwargs.get("shell") else cmd
     try:
-        kwargs.setdefault("shell", True)
-        proc = subprocess.Popen(cmd, **kwargs)
+        proc = subprocess.Popen(cmd_arg, **kwargs)
     except OSError as e:
         returncode, stdout_b, stderr_b = _oserror_to_output(e)
     else:

{salesforce_data_customcode-0.1.25 → salesforce_data_customcode-0.1.27}/src/datacustomcode/deploy.py

@@ -272,7 +272,7 @@ def create_deployment(
         raise
 
 
-PLATFORM_ENV_VAR = "DOCKER_DEFAULT_PLATFORM=linux/amd64"
+PLATFORM_ENV = {"DOCKER_DEFAULT_PLATFORM": "linux/amd64"}
 DOCKER_IMAGE_NAME = "datacloud-custom-code-dependency-builder"
 DEPENDENCIES_ARCHIVE_NAME = "native_dependencies"
 DEPENDENCIES_ARCHIVE_FULL_NAME = f"{DEPENDENCIES_ARCHIVE_NAME}.tar.gz"
@@ -286,19 +286,25 @@ def prepare_dependency_archive(directory: str, docker_network: str) -> None:
     cmd = f"docker images -q {DOCKER_IMAGE_NAME}"
     image_exists = cmd_output(cmd)
 
+    docker_env = {**os.environ, **PLATFORM_ENV}
+
     if not image_exists:
         logger.info(f"Building docker image with docker network: {docker_network}...")
         cmd = docker_build_cmd(docker_network)
-        cmd_output(cmd)
+        cmd_output(cmd, env=docker_env)
 
-    with tempfile.TemporaryDirectory() as temp_dir:
+    # ignore_cleanup_errors=True: on Windows, Docker creates files inside the
+    # mounted volume whose permissions prevent the host from deleting them.
+    # The archive has already been copied out, so silently skipping leftover
+    # files is safe and avoids a fatal error on context-manager exit.
+    with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as temp_dir:
         logger.info(
             f"Building dependencies archive with docker network: {docker_network}"
         )
         shutil.copy("requirements.txt", temp_dir)
         shutil.copy("build_native_dependencies.sh", temp_dir)
         cmd = docker_run_cmd(docker_network, temp_dir)
-        cmd_output(cmd)
+        cmd_output(cmd, env=docker_env)
         archives_temp_path = os.path.join(temp_dir, DEPENDENCIES_ARCHIVE_FULL_NAME)
         os.makedirs(os.path.dirname(DEPENDENCIES_ARCHIVE_PATH), exist_ok=True)
         shutil.copy(archives_temp_path, DEPENDENCIES_ARCHIVE_PATH)
@@ -307,10 +313,7 @@ def prepare_dependency_archive(directory: str, docker_network: str) -> None:
 
 
 def docker_build_cmd(network: str) -> str:
-    cmd = (
-        f"{PLATFORM_ENV_VAR} docker build -t {DOCKER_IMAGE_NAME} "
-        f"--file Dockerfile.dependencies . "
-    )
+    cmd = f"docker build -t {DOCKER_IMAGE_NAME} --file Dockerfile.dependencies . "
 
     if network != "default":
         cmd = cmd + f"--network {network}"
@@ -318,12 +321,11 @@ def docker_build_cmd(network: str) -> str:
     return cmd
 
 
-def docker_run_cmd(network: str, temp_dir) -> str:
-    cmd = (
-        f"{PLATFORM_ENV_VAR} docker run --rm "
-        f"-v {temp_dir}:/workspace "
-        f"{DOCKER_IMAGE_NAME} "
-    )
+def docker_run_cmd(network: str, temp_dir: str) -> str:
+    # Normalise path separators: Docker expects forward slashes even on Windows,
+    # and quoting handles paths that contain spaces.
+    docker_path = temp_dir.replace("\\", "/")
+    cmd = f'docker run --rm -v "{docker_path}:/workspace" {DOCKER_IMAGE_NAME} '
 
     if network != "default":
         cmd = cmd + f"--network {network} "

{salesforce_data_customcode-0.1.25/src/datacustomcode/templates/script → salesforce_data_customcode-0.1.27/src/datacustomcode/templates/function}/Dockerfile.dependencies

@@ -1,4 +1,4 @@
-FROM public.ecr.aws/emr-on-eks/spark/emr-7.3.0:latest
+FROM python:3.11-slim
 
 USER root
 

{salesforce_data_customcode-0.1.25 → salesforce_data_customcode-0.1.27}/src/datacustomcode/templates/function/payload/entrypoint.py

@@ -33,8 +33,8 @@ def chunk_text(text: str, chunk_size: int = 1000) -> List[str]:
     return chunks
 
 
-def dc_function(request: dict) -> dict:
-    logger.info("Inside DC Function")
+def function(request: dict) -> dict:
+    logger.info("Inside Function")
     logger.info(request)
 
     items = request["input"]
@@ -107,7 +107,7 @@ if __name__ == "__main__":
     }
 
     # Run the function
-    result = dc_function(test_request)
+    result = function(test_request)
 
     # Print the results in a more readable format
     print("\nChunking Results:")

{salesforce_data_customcode-0.1.25/src/datacustomcode/templates/script → salesforce_data_customcode-0.1.27/src/datacustomcode/templates/function}/requirements-dev.txt

@@ -1,10 +1,3 @@
 # Required packages for the project - Do not modify
-salesforce-cdp-connector>=1.0.16
-pyspark==3.5.1
-pandas
-numpy
 pydantic
-jupyterlab
-ipywidgets
-tqdm
 salesforce-data-customcode

salesforce_data_customcode-0.1.25/src/datacustomcode/templates/script/Dockerfile

@@ -1,18 +0,0 @@
-FROM public.ecr.aws/emr-on-eks/spark/emr-7.3.0:latest
-
-USER root
-
-# install from dev requirements.txt
-COPY requirements-dev.txt ./requirements-dev.txt
-RUN pip3.11 install --no-cache-dir -r requirements-dev.txt
-
-# Install from requirements.txt:
-COPY requirements.txt ./requirements.txt
-RUN pip3.11 install --no-cache-dir -r requirements.txt
-
-# Create workspace directory
-RUN mkdir /workspace
-
-# Set user and working directory
-USER hadoop:hadoop
-WORKDIR /workspace

salesforce_data_customcode-0.1.25/src/datacustomcode/templates/script/account.ipynb

@@ -1,86 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "0",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from datacustomcode.client import Client\n",
-    "from datacustomcode.io.writer.base import WriteMode\n",
-    "from pyspark.sql.functions import col, upper"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "1",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "client = Client()\n",
-    "\n",
-    "df = client.read_dlo(\"Account_std__dll\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "2",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Perform transformations on the DataFrame\n",
-    "df_upper1 = df.withColumn(\"Description__c\", upper(col(\"Description__c\")))\n",
-    "\n",
-    "# Drop specific columns related to relationships\n",
-    "df_upper1 = df_upper1.drop(\"SfdcOrganizationId__c\")\n",
-    "df_upper1 = df_upper1.drop(\"KQ_Id__c\")\n",
-    "\n",
-    "df_upper1.show()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "3",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Save the transformed DataFrame\n",
-    "dlo_name = \"Account_std_copy__dll\"\n",
-    "client.write_to_dlo(dlo_name, df_upper1, write_mode=WriteMode.APPEND)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "4",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.11.11"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}

salesforce_data_customcode-0.1.25/src/datacustomcode/templates/script/examples/employee_hierarchy/employee_data.csv

@@ -1,13 +0,0 @@
-id,name,position,manager_id
-1,Alice,CEO,
-2,Bob,VP Engineering,1
-3,Charlie,VP Sales,1
-4,David,Engineering Lead,2
-5,Eve,Engineering Lead,2
-6,Frank,Sales Manager,3
-7,Grace,Software Eng.,4
-8,Hannah,Software Eng.,4
-9,Ian,Software Eng.,5
-10,Jack,Sales Rep,6
-11,Kelly,Sales Rep,6
-12,Leo,Intern,7

salesforce_data_customcode-0.1.25/src/datacustomcode/templates/script/examples/employee_hierarchy/entrypoint.py

@@ -1,78 +0,0 @@
-from pyspark.sql.functions import (
-    col,
-    concat_ws,
-    lit,
-)
-
-from datacustomcode.client import Client
-from datacustomcode.io.writer.base import WriteMode
-
-
-def main():
-    client = Client()
-
-    employees = client.read_dlo("Employee__dll").persist()
-    employees = employees.select("id__c", "manager_id__c", "name__c", "position__c")
-    employees.show()
-    employees_with_manager = (
-        employees.alias("e")
-        .join(
-            employees.alias("m"),
-            col("e.manager_id__c").cast("string") == col("m.id__c").cast("string"),
-            "left",
-        )
-        .select(
-            col("e.id__c"),
-            col("e.name__c"),
-            col("e.position__c"),
-            col("e.manager_id__c"),
-            col("m.name__c").alias("manager_name__c"),
-        )
-        .persist()
-    )
-
-    hierarchy_df = (
-        employees_with_manager.filter(col("manager_id__c").isNull())
-        .withColumn("hierarchy_level__c", lit(1))
-        .withColumn("management_chain__c", col("name__c"))
-        .persist()
-    )
-
-    current_level = 1
-
-    while True:
-        ewm = employees_with_manager.alias("ewm")
-        hdf = hierarchy_df.filter(col("hierarchy_level__c") == current_level).alias(
-            "hdf"
-        )
-
-        next_level_df = ewm.join(
-            hdf,
-            col("ewm.manager_id__c").cast("string") == col("hdf.id__c").cast("string"),
-            "inner",
-        ).select(
-            col("ewm.id__c"),
-            col("ewm.name__c"),
-            col("ewm.position__c"),
-            col("ewm.manager_id__c"),
-            col("ewm.manager_name__c"),
-            (col("hdf.hierarchy_level__c") + 1).alias("hierarchy_level__c"),
-            concat_ws(" | ", col("hdf.management_chain__c"), col("ewm.name__c")).alias(
-                "management_chain__c"
-            ),
-        )
-
-        if next_level_df.isEmpty():
-            break
-
-        hierarchy_df = hierarchy_df.union(next_level_df).persist()
-        current_level += 1
-
-    hierarchy_df = hierarchy_df.orderBy("hierarchy_level__c", "manager_id__c", "id__c")
-
-    dlo_name = "Employee_Hierarchy__dll"
-    client.write_to_dlo(dlo_name, hierarchy_df, WriteMode.APPEND)
-
-
-if __name__ == "__main__":
-    main()

salesforce_data_customcode-0.1.25/src/datacustomcode/templates/script/jupyterlab.sh

@@ -1,97 +0,0 @@
-#!/bin/bash
-
-# Description: This script is used to start/stop the jupyter notebook in a docker container
-
-# Function to open browser based on OS
-open_browser() {
-    local url=$1
-    case "$(uname -s)" in
-        Darwin*)  # macOS
-            open "$url"
-            ;;
-        Linux*)   # Linux
-            if command -v xdg-open &> /dev/null; then
-                xdg-open "$url"
-            elif command -v gnome-open &> /dev/null; then
-                gnome-open "$url"
-            else
-                echo "Could not detect the web browser to use"
-                return 1
-            fi
-            ;;
-        CYGWIN*|MINGW32*|MSYS*|MINGW*)  # Windows
-            start "$url"
-            ;;
-        *)
-            echo "Unknown operating system"
-            return 1
-            ;;
-    esac
-}
-
-# Function to check if Docker is installed and running
-check_docker() {
-    if ! command -v docker &> /dev/null; then
-        echo "Docker is not installed"
-        exit 1
-    fi
-    echo "Docker is installed"
-    docker --version
-
-    if ! docker info &> /dev/null; then
-        echo "Docker daemon is not running"
-        exit 1
-    fi
-    echo "Docker daemon is running"
-}
-
-# Function to start Jupyter server
-start_jupyter() {
-    echo "Building the docker image"
-    docker build -t datacloud-customcode .
-
-    echo "Running the docker container"
-    docker run -d --rm -p 8888:8888 \
-        -v $(pwd):/workspace \
-        --name jupyter-server \
-        datacloud-customcode jupyter lab \
-        --ip=0.0.0.0 \
-        --port=8888 \
-        --no-browser \
-        --allow-root \
-        --NotebookApp.token='' \
-        --NotebookApp.password='' \
-        --notebook-dir=/workspace
-
-    sleep 3  # Wait for server to start
-    open_browser "http://localhost:8888"
-}
-
-# Function to stop Jupyter server
-stop_jupyter() {
-    echo "Stopping Jupyter server container..."
-    if docker ps -q --filter "name=jupyter-server" | grep -q .; then
-        docker stop jupyter-server
-        echo "Jupyter server stopped successfully"
-    else
-        echo "No Jupyter server container running"
-    fi
-}
-
-# Main script logic
-case "$1" in
-    "start")
-        check_docker
-        start_jupyter
-        ;;
-    "stop")
-        check_docker
-        stop_jupyter
-        ;;
-    *)
-        echo "Usage: $0 {start|stop}"
-        echo "  start - Start Jupyter server"
-        echo "  stop  - Stop Jupyter server"
-        exit 1
-        ;;
-esac