wherobots-python-sdk 0.2.1__tar.gz → 0.3.0__tar.gz

{wherobots_python_sdk-0.2.1 → wherobots_python_sdk-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: wherobots-python-sdk
-Version: 0.2.1
+Version: 0.3.0
 Summary: Python SDK for Wherobots (currently covers the Jobs REST API)
 Author-email: Wherobots <support@wherobots.com>
 License-Expression: Apache-2.0
@@ -22,6 +22,7 @@ Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: requests>=2.33.0
+Requires-Dist: boto3>=1.34.0
 Provides-Extra: dev
 Requires-Dist: pytest>=7.4.0; extra == "dev"
 Requires-Dist: pytest-cov>=4.1.0; extra == "dev"
@@ -187,7 +188,7 @@ WherobotsJob(
     spark_driver_disk_gb: int | None = None, # Driver disk size (GB)
     spark_executor_disk_gb: int | None = None, # Executor disk size (GB)
     jar_main_class: str | None = None,       # Main class (required for JARs)
-    auto_upload: bool = True,                # Upload local files to S3
+    auto_upload: bool = True,                # Upload local files to S3 (managed storage)
     base_url: str | None = None,             # Override API URL
     request_timeout_seconds: int | None = None,  # HTTP timeout
     config: WherobotsConfig | None = None,   # Full config object
@@ -235,7 +236,7 @@ with WherobotsJob(script="s3://bucket/script.py", name="my-job") as job:
 | `from_run_id(run_id, api_key=None, ...)` | `WherobotsJob` | Attach to an existing run for read-only log/metric access. No script required. `submit()` is disabled on the returned instance. |
 | `list_runs(...)` | `RunListPage` | List runs with optional filters. No instance required. |
 | `add_pypi_dependency(name, version)` | `dict` | Create a PyPI dependency dict for the `dependencies` parameter. |
-| `add_file_dependency(file_path)` | `dict` | Create a file dependency dict (`.jar`, `.whl`, `.zip`, `.json`). |
+| `add_file_dependency(file_path)` | `dict` | Create a file dependency dict (`.jar`, `.whl`, `.zip`, `.json`). Accepts an `s3://` URI or a local path — local paths are auto-uploaded to **managed storage** at `submit()` time. For non-managed destinations, upload via `FilesAPI` first and pass the resulting `s3://` URI. |
 
 #### Attaching to an Existing Run
 
@@ -307,6 +308,80 @@ job = WherobotsJob(
 )
 ```
 
+##### Local Dependency Files
+
+`add_file_dependency` also accepts a **local path** to a `.whl`, `.jar`, `.zip`,
+or `.json`. The SDK uploads the file just before the job is submitted, fetching
+short-lived AWS STS credentials from the Wherobots API (`POST /storage/credentials`)
+and using boto3 multipart under the hood — no AWS credentials need to leave your
+environment.
+
+```python
+job = WherobotsJob(
+    script="my_job.py",                  # local script auto-upload still works
+    name="job-with-local-wheel",
+    dependencies=[
+        # Local wheel — uploaded to managed storage at submit() time
+        WherobotsJob.add_file_dependency("dist/mypkg-0.1.0-py3-none-any.whl"),
+        # Already-uploaded artifact — used as-is
+        WherobotsJob.add_file_dependency("s3://bucket/shared-libs/lib-2.0.jar"),
+        WherobotsJob.add_pypi_dependency("pandas", "2.0.3"),
+    ],
+)
+job.submit()
+```
+
+Auto-upload always targets the org's **managed storage**:
+
+| Artifact | Destination |
+|---|---|
+| Script (when `script` is a local path) | `s3://<managed.path>/scripts/<filename>` |
+| Dependency (when path is a local file) | `s3://<managed.path>/dependencies/<filename>` |
+
+Local dependency uploads honor the same `auto_upload` flag as the script —
+if `auto_upload=False`, attempting to submit a job with a local dependency
+path raises `WherobotsValidationError`. The 5 GiB cap on a single upload
+applies; for larger artifacts, pre-upload and pass an `s3://` URI.
+
+### Uploading to a Storage Integration
+
+`WherobotsJob` stays focused on job orchestration — it doesn't take any
+"where files go" kwargs. To upload to a [Storage Integration](https://docs.wherobots.com/latest/develop/storage-management/storage/)
+or a custom subdirectory, use `FilesAPI` directly and pass the resulting
+`s3://` URI to `WherobotsJob`:
+
+```python
+from wherobots import FilesAPI, WherobotsJob
+from wherobots.config import WherobotsConfig
+
+with FilesAPI.from_config(WherobotsConfig.from_env()) as files:
+    si = files.resolve_integration("customer-bucket")
+    # Build explicit destinations; upload_file is the "upload this file
+    # to this exact s3 path" primitive.
+    script_uri = files.upload_file(
+        "my_job.py",
+        files.dest_uri_for(si, "my_job.py", "job-scripts/sdk-e2e"),
+    )
+    wheel_uri = files.upload_file(
+        "dist/internal-lib-1.0.whl",
+        files.dest_uri_for(si, "internal-lib-1.0.whl", "job-scripts/sdk-e2e"),
+    )
+
+job = WherobotsJob(
+    script=script_uri,                              # already-uploaded URI
+    name="job-in-integration",
+    dependencies=[WherobotsJob.add_file_dependency(wheel_uri)],
+)
+job.submit()
+```
+
+Both artifacts land under `s3://<integration.path>/job-scripts/sdk-e2e/`.
+
+`FilesAPI` is also where to look for low-level controls (custom managed
+subdirs, raw `s3://` destinations, STS credential fetch for BYO upload
+tooling). See `FilesAPI.upload_file`, `upload_managed_file`,
+`upload_dependency`, `dest_uri_for`, and `get_storage_credentials`.
+
 #### JAR Jobs
 
 ```python

{wherobots_python_sdk-0.2.1 → wherobots_python_sdk-0.3.0}/README.md

@@ -154,7 +154,7 @@ WherobotsJob(
     spark_driver_disk_gb: int | None = None, # Driver disk size (GB)
     spark_executor_disk_gb: int | None = None, # Executor disk size (GB)
     jar_main_class: str | None = None,       # Main class (required for JARs)
-    auto_upload: bool = True,                # Upload local files to S3
+    auto_upload: bool = True,                # Upload local files to S3 (managed storage)
     base_url: str | None = None,             # Override API URL
     request_timeout_seconds: int | None = None,  # HTTP timeout
     config: WherobotsConfig | None = None,   # Full config object
@@ -202,7 +202,7 @@ with WherobotsJob(script="s3://bucket/script.py", name="my-job") as job:
 | `from_run_id(run_id, api_key=None, ...)` | `WherobotsJob` | Attach to an existing run for read-only log/metric access. No script required. `submit()` is disabled on the returned instance. |
 | `list_runs(...)` | `RunListPage` | List runs with optional filters. No instance required. |
 | `add_pypi_dependency(name, version)` | `dict` | Create a PyPI dependency dict for the `dependencies` parameter. |
-| `add_file_dependency(file_path)` | `dict` | Create a file dependency dict (`.jar`, `.whl`, `.zip`, `.json`). |
+| `add_file_dependency(file_path)` | `dict` | Create a file dependency dict (`.jar`, `.whl`, `.zip`, `.json`). Accepts an `s3://` URI or a local path — local paths are auto-uploaded to **managed storage** at `submit()` time. For non-managed destinations, upload via `FilesAPI` first and pass the resulting `s3://` URI. |
 
 #### Attaching to an Existing Run
 
@@ -274,6 +274,80 @@ job = WherobotsJob(
 )
 ```
 
+##### Local Dependency Files
+
+`add_file_dependency` also accepts a **local path** to a `.whl`, `.jar`, `.zip`,
+or `.json`. The SDK uploads the file just before the job is submitted, fetching
+short-lived AWS STS credentials from the Wherobots API (`POST /storage/credentials`)
+and using boto3 multipart under the hood — no AWS credentials need to leave your
+environment.
+
+```python
+job = WherobotsJob(
+    script="my_job.py",                  # local script auto-upload still works
+    name="job-with-local-wheel",
+    dependencies=[
+        # Local wheel — uploaded to managed storage at submit() time
+        WherobotsJob.add_file_dependency("dist/mypkg-0.1.0-py3-none-any.whl"),
+        # Already-uploaded artifact — used as-is
+        WherobotsJob.add_file_dependency("s3://bucket/shared-libs/lib-2.0.jar"),
+        WherobotsJob.add_pypi_dependency("pandas", "2.0.3"),
+    ],
+)
+job.submit()
+```
+
+Auto-upload always targets the org's **managed storage**:
+
+| Artifact | Destination |
+|---|---|
+| Script (when `script` is a local path) | `s3://<managed.path>/scripts/<filename>` |
+| Dependency (when path is a local file) | `s3://<managed.path>/dependencies/<filename>` |
+
+Local dependency uploads honor the same `auto_upload` flag as the script —
+if `auto_upload=False`, attempting to submit a job with a local dependency
+path raises `WherobotsValidationError`. The 5 GiB cap on a single upload
+applies; for larger artifacts, pre-upload and pass an `s3://` URI.
+
+### Uploading to a Storage Integration
+
+`WherobotsJob` stays focused on job orchestration — it doesn't take any
+"where files go" kwargs. To upload to a [Storage Integration](https://docs.wherobots.com/latest/develop/storage-management/storage/)
+or a custom subdirectory, use `FilesAPI` directly and pass the resulting
+`s3://` URI to `WherobotsJob`:
+
+```python
+from wherobots import FilesAPI, WherobotsJob
+from wherobots.config import WherobotsConfig
+
+with FilesAPI.from_config(WherobotsConfig.from_env()) as files:
+    si = files.resolve_integration("customer-bucket")
+    # Build explicit destinations; upload_file is the "upload this file
+    # to this exact s3 path" primitive.
+    script_uri = files.upload_file(
+        "my_job.py",
+        files.dest_uri_for(si, "my_job.py", "job-scripts/sdk-e2e"),
+    )
+    wheel_uri = files.upload_file(
+        "dist/internal-lib-1.0.whl",
+        files.dest_uri_for(si, "internal-lib-1.0.whl", "job-scripts/sdk-e2e"),
+    )
+
+job = WherobotsJob(
+    script=script_uri,                              # already-uploaded URI
+    name="job-in-integration",
+    dependencies=[WherobotsJob.add_file_dependency(wheel_uri)],
+)
+job.submit()
+```
+
+Both artifacts land under `s3://<integration.path>/job-scripts/sdk-e2e/`.
+
+`FilesAPI` is also where to look for low-level controls (custom managed
+subdirs, raw `s3://` destinations, STS credential fetch for BYO upload
+tooling). See `FilesAPI.upload_file`, `upload_managed_file`,
+`upload_dependency`, `dest_uri_for`, and `get_storage_credentials`.
+
 #### JAR Jobs
 
 ```python

{wherobots_python_sdk-0.2.1 → wherobots_python_sdk-0.3.0}/pyproject.toml

@@ -26,6 +26,7 @@ classifiers = [
 keywords = ["wherobots", "spark", "sedona", "geospatial", "jobs", "sdk", "rest-api"]
 dependencies = [
     "requests>=2.33.0",
+    "boto3>=1.34.0",
 ]
 
 [project.urls]
@@ -85,6 +86,13 @@ show_error_codes = true
 module = "tests.*"
 ignore_errors = true
 
+[[tool.mypy.overrides]]
+# boto3 / botocore ship runtime types in some versions but the
+# pre-commit mypy env doesn't install them. Declare the policy once
+# here rather than scattering per-import ``# type: ignore`` comments.
+module = ["boto3", "boto3.*", "botocore", "botocore.*"]
+ignore_missing_imports = true
+
 [tool.pytest.ini_options]
 testpaths = ["tests"]
 python_files = ["test_*.py"]

{wherobots_python_sdk-0.2.1 → wherobots_python_sdk-0.3.0}/tests/test_client.py

@@ -329,7 +329,7 @@ class TestPresignedUpload:
     def test_presigned_happy_path(self, mock_from_config, mock_env):
         """Presigned upload succeeds via _prepare_script_uri()."""
         mock_files_api = MagicMock()
-        mock_files_api.upload_file.return_value = (
+        mock_files_api.upload_managed_file.return_value = (
             "s3://managed-bucket/org/data/shared/scripts/my_script.py"
         )
         mock_from_config.return_value = mock_files_api
@@ -342,14 +342,14 @@ class TestPresignedUpload:
         uri = job._prepare_script_uri()
 
         assert uri == "s3://managed-bucket/org/data/shared/scripts/my_script.py"
-        mock_files_api.upload_file.assert_called_once_with("my_script.py")
+        mock_files_api.upload_managed_file.assert_called_once_with("my_script.py", subdir="scripts")
         mock_files_api.close.assert_called_once()
 
     @patch("wherobots.api.files.FilesAPI.from_config")
     def test_presigned_success_caches_uri(self, mock_from_config, mock_env):
         """_prepare_script_uri() caches the result and doesn't re-upload."""
         mock_files_api = MagicMock()
-        mock_files_api.upload_file.return_value = (
+        mock_files_api.upload_managed_file.return_value = (
             "s3://managed-bucket/org/data/shared/scripts/my_script.py"
         )
         mock_from_config.return_value = mock_files_api
@@ -363,14 +363,16 @@ class TestPresignedUpload:
         uri2 = job._prepare_script_uri()
 
         assert uri1 == uri2
-        # upload_file should only be called once (cached on second call)
-        mock_files_api.upload_file.assert_called_once()
+        # upload_managed_file should only be called once (cached on second call)
+        mock_files_api.upload_managed_file.assert_called_once()
 
     @patch("wherobots.api.files.FilesAPI.from_config")
     def test_presigned_fails_raises_directly(self, mock_from_config, mock_env):
         """Presigned upload fails → WherobotsAPIError raised with no fallback."""
         mock_files_api = MagicMock()
-        mock_files_api.upload_file.side_effect = WherobotsAPIError("presigned 500", status_code=500)
+        mock_files_api.upload_managed_file.side_effect = WherobotsAPIError(
+            "presigned 500", status_code=500
+        )
         mock_from_config.return_value = mock_files_api
 
         job = WherobotsJob(
@@ -387,7 +389,7 @@ class TestPresignedUpload:
     def test_presigned_closes_files_api_on_success(self, mock_from_config, mock_env):
         """FilesAPI.close() is called even when upload succeeds."""
         mock_files_api = MagicMock()
-        mock_files_api.upload_file.return_value = "s3://bucket/key"
+        mock_files_api.upload_managed_file.return_value = "s3://bucket/key"
         mock_from_config.return_value = mock_files_api
 
         job = WherobotsJob(
@@ -403,7 +405,7 @@ class TestPresignedUpload:
     def test_presigned_closes_files_api_on_failure(self, mock_from_config, mock_env):
         """FilesAPI.close() is called even when upload fails."""
         mock_files_api = MagicMock()
-        mock_files_api.upload_file.side_effect = WherobotsAPIError("boom")
+        mock_files_api.upload_managed_file.side_effect = WherobotsAPIError("boom")
         mock_from_config.return_value = mock_files_api
 
         job = WherobotsJob(
@@ -665,3 +667,157 @@ class TestPollForLogsDrain:
         job.poll_for_logs(follow=False, log_handler=captured.append)
 
         assert job._api.get_logs.call_count == 1
+
+
+# ── add_file_dependency local-path / lazy upload ─────────────────────────
+
+
+class TestAddFileDependency:
+    def test_s3_uri_unchanged(self):
+        dep = WherobotsJob.add_file_dependency("s3://bucket/lib.whl")
+        assert dep == {"sourceType": "FILE", "filePath": "s3://bucket/lib.whl"}
+        assert "_pendingUpload" not in dep
+
+    def test_local_path_marks_pending(self):
+        dep = WherobotsJob.add_file_dependency("dist/lib.whl")
+        assert dep["filePath"] == "dist/lib.whl"
+        assert dep["_pendingUpload"] is True
+        # Storage-integration routing belongs to FilesAPI now — sentinel
+        # keys for SI / dest_subdir are not present.
+        assert "_storageIntegration" not in dep
+        assert "_destSubdir" not in dep
+
+    def test_bad_extension_raises(self):
+        with pytest.raises(WherobotsValidationError, match="extensions"):
+            WherobotsJob.add_file_dependency("dist/lib.tar.gz")
+
+
+class TestPrepareDependencies:
+    @patch("wherobots.api.files.FilesAPI.from_config")
+    def test_local_dep_is_uploaded_and_sentinels_stripped(self, mock_from_config, mock_env):
+        mock_files = MagicMock()
+        mock_files.upload_dependency.return_value = "s3://managed/org/dependencies/lib.whl"
+        mock_from_config.return_value = mock_files
+
+        job = WherobotsJob(
+            script="s3://bucket/main.py",
+            name="local-dep-job",
+            runtime="tiny",
+            dependencies=[
+                WherobotsJob.add_file_dependency("dist/lib.whl"),
+                WherobotsJob.add_pypi_dependency("pandas", "2.0.3"),
+            ],
+        )
+        job._prepare_dependencies()
+
+        # First dep resolved + sentinel keys gone.
+        local_dep = job.dependencies[0]
+        assert local_dep["filePath"] == "s3://managed/org/dependencies/lib.whl"
+        assert "_pendingUpload" not in local_dep
+        assert "_storageIntegration" not in local_dep
+        # PyPI dep untouched.
+        assert job.dependencies[1] == {
+            "sourceType": "PYPI",
+            "libraryName": "pandas",
+            "libraryVersion": "2.0.3",
+        }
+        mock_files.upload_dependency.assert_called_once_with("dist/lib.whl")
+        mock_files.close.assert_called_once()
+
+    def test_no_pending_deps_skips_files_api(self, mock_env):
+        """If everything is already s3://, no FilesAPI session is created."""
+        with patch("wherobots.api.files.FilesAPI.from_config") as mock_from_config:
+            job = WherobotsJob(
+                script="s3://bucket/main.py",
+                name="all-s3-deps",
+                runtime="tiny",
+                dependencies=[
+                    WherobotsJob.add_file_dependency("s3://b/a.whl"),
+                    WherobotsJob.add_pypi_dependency("x", "1"),
+                ],
+            )
+            job._prepare_dependencies()
+            mock_from_config.assert_not_called()
+
+    @patch("wherobots.api.files.FilesAPI.from_config")
+    def test_partial_failure_leaves_dependencies_untouched(self, mock_from_config, mock_env):
+        """If upload_dependency raises mid-list, ``self.dependencies``
+        stays in its original sentinel-tagged form so a retry runs the
+        same work cleanly. No partially-resolved state ships."""
+        mock_files = MagicMock()
+        mock_files.upload_dependency.side_effect = [
+            "s3://managed/dependencies/first.whl",
+            WherobotsAPIError("upload boom"),
+        ]
+        mock_from_config.return_value = mock_files
+
+        job = WherobotsJob(
+            script="s3://bucket/main.py",
+            name="partial-failure-job",
+            runtime="tiny",
+            dependencies=[
+                WherobotsJob.add_file_dependency("dist/first.whl"),
+                WherobotsJob.add_file_dependency("dist/second.whl"),
+            ],
+        )
+        before = [dict(d) for d in job.dependencies]
+
+        with pytest.raises(WherobotsAPIError, match="upload boom"):
+            job._prepare_dependencies()
+
+        # Both sentinel-tagged dicts still untouched on the instance.
+        assert job.dependencies == before
+        assert all(d.get("_pendingUpload") is True for d in job.dependencies)
+
+    def test_auto_upload_false_raises_on_local_dep(self, mock_env):
+        job = WherobotsJob(
+            script="s3://bucket/main.py",
+            name="no-auto-upload",
+            runtime="tiny",
+            auto_upload=False,
+            dependencies=[WherobotsJob.add_file_dependency("dist/lib.whl")],
+        )
+        with pytest.raises(WherobotsValidationError, match="auto_upload"):
+            job._prepare_dependencies()
+
+    @patch("wherobots.api.files.FilesAPI.from_config")
+    def test_build_payload_runs_dependency_prep(self, mock_from_config, mock_env):
+        """_build_payload must call _prepare_dependencies before serializing."""
+        mock_files = MagicMock()
+        mock_files.upload_dependency.return_value = "s3://managed/dependencies/lib.whl"
+        mock_from_config.return_value = mock_files
+
+        job = WherobotsJob(
+            script="s3://bucket/main.py",
+            name="build-payload-job",
+            runtime="tiny",
+            dependencies=[WherobotsJob.add_file_dependency("dist/lib.whl")],
+        )
+        payload = job._build_payload()
+
+        # The CreateRunPayload must serialize cleanly — no sentinel keys leak.
+        body = payload.to_dict()
+        env = body["environment"]
+        assert env["dependencies"] == [
+            {"sourceType": "FILE", "filePath": "s3://managed/dependencies/lib.whl"}
+        ]
+
+
+# ── Script auto-upload stays scoped to managed storage ────────────────
+
+
+class TestScriptAutoUpload:
+    @patch("wherobots.api.files.FilesAPI.from_config")
+    def test_script_auto_upload_routes_to_managed(self, mock_from_config, mock_env):
+        """``WherobotsJob`` auto-upload always targets managed storage.
+        Integration uploads are an explicit ``FilesAPI`` step."""
+        mock_files = MagicMock()
+        mock_files.upload_managed_file.return_value = "s3://managed/scripts/job.py"
+        mock_from_config.return_value = mock_files
+
+        job = WherobotsJob(script="job.py", name="default-script", runtime="tiny")
+        uri = job._prepare_script_uri()
+
+        assert uri == "s3://managed/scripts/job.py"
+        mock_files.upload_managed_file.assert_called_once_with("job.py", subdir="scripts")
+        mock_files.upload_dependency.assert_not_called()