oleanderhq-sdk 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- oleanderhq_sdk-0.1.0/.gitignore +6 -0
- oleanderhq_sdk-0.1.0/PKG-INFO +121 -0
- oleanderhq_sdk-0.1.0/README.md +106 -0
- oleanderhq_sdk-0.1.0/pyproject.toml +30 -0
- oleanderhq_sdk-0.1.0/src/oleander_sdk/__init__.py +48 -0
- oleanderhq_sdk-0.1.0/src/oleander_sdk/client.py +244 -0
- oleanderhq_sdk-0.1.0/src/oleander_sdk/models.py +165 -0
- oleanderhq_sdk-0.1.0/src/oleander_sdk/py.typed +0 -0
- oleanderhq_sdk-0.1.0/tests/__init__.py +0 -0
- oleanderhq_sdk-0.1.0/tests/test_client.py +115 -0
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: oleanderhq-sdk
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Python SDK for oleander: query lake, list and launch Spark jobs
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Keywords: data,lake,oleander,openLineage,query,sdk,spark
|
|
7
|
+
Requires-Python: >=3.10
|
|
8
|
+
Requires-Dist: httpx<1,>=0.27
|
|
9
|
+
Requires-Dist: pydantic<3,>=2.7
|
|
10
|
+
Provides-Extra: dev
|
|
11
|
+
Requires-Dist: pytest-asyncio<1,>=0.24; extra == 'dev'
|
|
12
|
+
Requires-Dist: pytest<9,>=8; extra == 'dev'
|
|
13
|
+
Requires-Dist: respx<1,>=0.22; extra == 'dev'
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
|
|
16
|
+
# oleander Python SDK
|
|
17
|
+
|
|
18
|
+
Use the [oleander](https://oleander.dev) API from Python: run lake queries, list Spark jobs, and launch Spark jobs.
|
|
19
|
+
|
|
20
|
+
## Install
|
|
21
|
+
|
|
22
|
+
```bash
|
|
23
|
+
pip install oleanderhq-sdk
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
## Get your API key
|
|
27
|
+
|
|
28
|
+
Create an API key in [oleander settings](https://oleander.dev/app/settings), or run `oleander configure` if you use the CLI. You can pass the key when creating the client or set the `OLEANDER_API_KEY` environment variable.
|
|
29
|
+
|
|
30
|
+
## Quick start
|
|
31
|
+
|
|
32
|
+
```python
|
|
33
|
+
import asyncio
|
|
34
|
+
from oleander_sdk import Oleander
|
|
35
|
+
|
|
36
|
+
async def main():
|
|
37
|
+
oleander = Oleander()
|
|
38
|
+
result = await oleander.list_spark_jobs()
|
|
39
|
+
print(result.scripts)
|
|
40
|
+
|
|
41
|
+
asyncio.run(main())
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
## API
|
|
45
|
+
|
|
46
|
+
All methods are async. Use `await` when calling them.
|
|
47
|
+
|
|
48
|
+
### Query (lake)
|
|
49
|
+
|
|
50
|
+
Run a SQL query against the oleander lake. The second parameter is optional; `save` defaults to `False`. Use `save=True` to persist results as a table.
|
|
51
|
+
|
|
52
|
+
```python
|
|
53
|
+
from oleander_sdk import Oleander, QueryOptions
|
|
54
|
+
|
|
55
|
+
oleander = Oleander()
|
|
56
|
+
result = await oleander.query(
|
|
57
|
+
"SELECT * FROM public.iris_dataset LIMIT 10",
|
|
58
|
+
)
|
|
59
|
+
print(result.results.columns, result.results.rows)
|
|
60
|
+
print(result.row_count, result.execution_time)
|
|
61
|
+
if result.saved_table_name:
|
|
62
|
+
print("Saved to:", result.saved_table_name)
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
### List Spark jobs
|
|
66
|
+
|
|
67
|
+
Paginated list of your Spark job scripts. Options: `limit` (default 20), `offset` (default 0).
|
|
68
|
+
|
|
69
|
+
```python
|
|
70
|
+
from oleander_sdk import Oleander, ListSparkJobsOptions
|
|
71
|
+
|
|
72
|
+
oleander = Oleander()
|
|
73
|
+
result = await oleander.list_spark_jobs()
|
|
74
|
+
print(result.scripts, result.has_more)
|
|
75
|
+
|
|
76
|
+
next_page = await oleander.list_spark_jobs(ListSparkJobsOptions(offset=20))
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
### Launch Spark job
|
|
80
|
+
|
|
81
|
+
Submit a Spark job. Required: `namespace`, `name`, `script_name`. Other options (e.g. `args`, machine types, tags) have defaults.
|
|
82
|
+
|
|
83
|
+
```python
|
|
84
|
+
from oleander_sdk import Oleander, SparkJobSubmitOptions
|
|
85
|
+
|
|
86
|
+
oleander = Oleander()
|
|
87
|
+
result = await oleander.submit_spark_job(SparkJobSubmitOptions(
|
|
88
|
+
namespace="my-namespace",
|
|
89
|
+
name="my-job-name",
|
|
90
|
+
script_name="my_script.py",
|
|
91
|
+
))
|
|
92
|
+
print("Run ID:", result.run_id)
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
### Wait for a run to finish
|
|
96
|
+
|
|
97
|
+
Submit and poll until the run completes. Optional: `poll_interval_ms` (default 10000), `timeout_ms` (default 600000).
|
|
98
|
+
|
|
99
|
+
```python
|
|
100
|
+
from oleander_sdk import Oleander, SubmitSparkJobAndWaitOptions
|
|
101
|
+
|
|
102
|
+
oleander = Oleander()
|
|
103
|
+
result = await oleander.submit_spark_job_and_wait(SubmitSparkJobAndWaitOptions(
|
|
104
|
+
namespace="my-namespace",
|
|
105
|
+
name="my-job-name",
|
|
106
|
+
script_name="my_script.py",
|
|
107
|
+
))
|
|
108
|
+
print(result.run_id, result.state) # COMPLETE | FAIL | ABORT
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
### Get run status
|
|
112
|
+
|
|
113
|
+
```python
|
|
114
|
+
run = await oleander.get_run(run_id)
|
|
115
|
+
print(run.state, run.duration)
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
## Options
|
|
119
|
+
|
|
120
|
+
- **Constructor**: `Oleander(api_key=..., base_url=...)`. Omit `api_key` to use `OLEANDER_API_KEY`. Set `base_url` to use a different endpoint (e.g. `http://localhost:3000`).
|
|
121
|
+
- **Models**: The package exports Pydantic models (e.g. `OleanderOptions`, `SparkJobSubmitOptions`) if you want to validate config or options yourself.
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
# oleander Python SDK
|
|
2
|
+
|
|
3
|
+
Use the [oleander](https://oleander.dev) API from Python: run lake queries, list Spark jobs, and launch Spark jobs.
|
|
4
|
+
|
|
5
|
+
## Install
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
pip install oleanderhq-sdk
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
## Get your API key
|
|
12
|
+
|
|
13
|
+
Create an API key in [oleander settings](https://oleander.dev/app/settings), or run `oleander configure` if you use the CLI. You can pass the key when creating the client or set the `OLEANDER_API_KEY` environment variable.
|
|
14
|
+
|
|
15
|
+
## Quick start
|
|
16
|
+
|
|
17
|
+
```python
|
|
18
|
+
import asyncio
|
|
19
|
+
from oleander_sdk import Oleander
|
|
20
|
+
|
|
21
|
+
async def main():
|
|
22
|
+
oleander = Oleander()
|
|
23
|
+
result = await oleander.list_spark_jobs()
|
|
24
|
+
print(result.scripts)
|
|
25
|
+
|
|
26
|
+
asyncio.run(main())
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
## API
|
|
30
|
+
|
|
31
|
+
All methods are async. Use `await` when calling them.
|
|
32
|
+
|
|
33
|
+
### Query (lake)
|
|
34
|
+
|
|
35
|
+
Run a SQL query against the oleander lake. The second parameter is optional; `save` defaults to `False`. Use `save=True` to persist results as a table.
|
|
36
|
+
|
|
37
|
+
```python
|
|
38
|
+
from oleander_sdk import Oleander, QueryOptions
|
|
39
|
+
|
|
40
|
+
oleander = Oleander()
|
|
41
|
+
result = await oleander.query(
|
|
42
|
+
"SELECT * FROM public.iris_dataset LIMIT 10",
|
|
43
|
+
)
|
|
44
|
+
print(result.results.columns, result.results.rows)
|
|
45
|
+
print(result.row_count, result.execution_time)
|
|
46
|
+
if result.saved_table_name:
|
|
47
|
+
print("Saved to:", result.saved_table_name)
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
### List Spark jobs
|
|
51
|
+
|
|
52
|
+
Paginated list of your Spark job scripts. Options: `limit` (default 20), `offset` (default 0).
|
|
53
|
+
|
|
54
|
+
```python
|
|
55
|
+
from oleander_sdk import Oleander, ListSparkJobsOptions
|
|
56
|
+
|
|
57
|
+
oleander = Oleander()
|
|
58
|
+
result = await oleander.list_spark_jobs()
|
|
59
|
+
print(result.scripts, result.has_more)
|
|
60
|
+
|
|
61
|
+
next_page = await oleander.list_spark_jobs(ListSparkJobsOptions(offset=20))
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
### Launch Spark job
|
|
65
|
+
|
|
66
|
+
Submit a Spark job. Required: `namespace`, `name`, `script_name`. Other options (e.g. `args`, machine types, tags) have defaults.
|
|
67
|
+
|
|
68
|
+
```python
|
|
69
|
+
from oleander_sdk import Oleander, SparkJobSubmitOptions
|
|
70
|
+
|
|
71
|
+
oleander = Oleander()
|
|
72
|
+
result = await oleander.submit_spark_job(SparkJobSubmitOptions(
|
|
73
|
+
namespace="my-namespace",
|
|
74
|
+
name="my-job-name",
|
|
75
|
+
script_name="my_script.py",
|
|
76
|
+
))
|
|
77
|
+
print("Run ID:", result.run_id)
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
### Wait for a run to finish
|
|
81
|
+
|
|
82
|
+
Submit and poll until the run completes. Optional: `poll_interval_ms` (default 10000), `timeout_ms` (default 600000).
|
|
83
|
+
|
|
84
|
+
```python
|
|
85
|
+
from oleander_sdk import Oleander, SubmitSparkJobAndWaitOptions
|
|
86
|
+
|
|
87
|
+
oleander = Oleander()
|
|
88
|
+
result = await oleander.submit_spark_job_and_wait(SubmitSparkJobAndWaitOptions(
|
|
89
|
+
namespace="my-namespace",
|
|
90
|
+
name="my-job-name",
|
|
91
|
+
script_name="my_script.py",
|
|
92
|
+
))
|
|
93
|
+
print(result.run_id, result.state) # COMPLETE | FAIL | ABORT
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
### Get run status
|
|
97
|
+
|
|
98
|
+
```python
|
|
99
|
+
run = await oleander.get_run(run_id)
|
|
100
|
+
print(run.state, run.duration)
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
## Options
|
|
104
|
+
|
|
105
|
+
- **Constructor**: `Oleander(api_key=..., base_url=...)`. Omit `api_key` to use `OLEANDER_API_KEY`. Set `base_url` to use a different endpoint (e.g. `http://localhost:3000`).
|
|
106
|
+
- **Models**: The package exports Pydantic models (e.g. `OleanderOptions`, `SparkJobSubmitOptions`) if you want to validate config or options yourself.
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "oleanderhq-sdk"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Python SDK for oleander: query lake, list and launch Spark jobs"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
requires-python = ">=3.10"
|
|
12
|
+
keywords = ["oleander", "sdk", "lake", "spark", "query", "data", "openLineage"]
|
|
13
|
+
dependencies = [
|
|
14
|
+
"httpx>=0.27,<1",
|
|
15
|
+
"pydantic>=2.7,<3",
|
|
16
|
+
]
|
|
17
|
+
|
|
18
|
+
[project.optional-dependencies]
|
|
19
|
+
dev = [
|
|
20
|
+
"pytest>=8,<9",
|
|
21
|
+
"pytest-asyncio>=0.24,<1",
|
|
22
|
+
"respx>=0.22,<1",
|
|
23
|
+
]
|
|
24
|
+
|
|
25
|
+
[tool.hatch.build.targets.wheel]
|
|
26
|
+
packages = ["src/oleander_sdk"]
|
|
27
|
+
|
|
28
|
+
[tool.pytest.ini_options]
|
|
29
|
+
asyncio_mode = "auto"
|
|
30
|
+
testpaths = ["tests"]
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""oleander Python SDK — query lake, list and launch Spark jobs."""
|
|
2
|
+
|
|
3
|
+
from .client import Oleander, SubmitAndWaitResult
|
|
4
|
+
from .models import (
|
|
5
|
+
ApiErrorBody,
|
|
6
|
+
DEFAULT_BASE_URL,
|
|
7
|
+
LakeQueryResult,
|
|
8
|
+
ListSparkJobsOptions,
|
|
9
|
+
ListSparkJobsResult,
|
|
10
|
+
OleanderOptions,
|
|
11
|
+
QueryOptions,
|
|
12
|
+
QueryResultColumns,
|
|
13
|
+
RunJobInfo,
|
|
14
|
+
RunPipelineInfo,
|
|
15
|
+
RunResponse,
|
|
16
|
+
RunState,
|
|
17
|
+
RunTag,
|
|
18
|
+
SparkJobListPage,
|
|
19
|
+
SparkJobRun,
|
|
20
|
+
SparkJobSubmitOptions,
|
|
21
|
+
SparkMachineType,
|
|
22
|
+
SubmitSparkJobAndWaitOptions,
|
|
23
|
+
TERMINAL_STATES,
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
__all__ = [
|
|
27
|
+
"Oleander",
|
|
28
|
+
"SubmitAndWaitResult",
|
|
29
|
+
"ApiErrorBody",
|
|
30
|
+
"DEFAULT_BASE_URL",
|
|
31
|
+
"LakeQueryResult",
|
|
32
|
+
"ListSparkJobsOptions",
|
|
33
|
+
"ListSparkJobsResult",
|
|
34
|
+
"OleanderOptions",
|
|
35
|
+
"QueryOptions",
|
|
36
|
+
"QueryResultColumns",
|
|
37
|
+
"RunJobInfo",
|
|
38
|
+
"RunPipelineInfo",
|
|
39
|
+
"RunResponse",
|
|
40
|
+
"RunState",
|
|
41
|
+
"RunTag",
|
|
42
|
+
"SparkJobListPage",
|
|
43
|
+
"SparkJobRun",
|
|
44
|
+
"SparkJobSubmitOptions",
|
|
45
|
+
"SparkMachineType",
|
|
46
|
+
"SubmitSparkJobAndWaitOptions",
|
|
47
|
+
"TERMINAL_STATES",
|
|
48
|
+
]
|
|
@@ -0,0 +1,244 @@
|
|
|
1
|
+
"""Oleander API client (mirrors client.ts)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import os
|
|
7
|
+
import time
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
from typing import Optional
|
|
10
|
+
|
|
11
|
+
import httpx
|
|
12
|
+
|
|
13
|
+
from .models import (
|
|
14
|
+
ApiErrorBody,
|
|
15
|
+
DEFAULT_BASE_URL,
|
|
16
|
+
LakeQueryResult,
|
|
17
|
+
ListSparkJobsOptions,
|
|
18
|
+
ListSparkJobsResult,
|
|
19
|
+
OleanderOptions,
|
|
20
|
+
QueryOptions,
|
|
21
|
+
RunResponse,
|
|
22
|
+
SparkJobRun,
|
|
23
|
+
SparkJobSubmitOptions,
|
|
24
|
+
SubmitSparkJobAndWaitOptions,
|
|
25
|
+
TERMINAL_STATES,
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _parse_error_body(body: dict, status: int) -> str:
|
|
30
|
+
try:
|
|
31
|
+
parsed = ApiErrorBody.model_validate(body)
|
|
32
|
+
return parsed.error or parsed.details or f"HTTP {status}"
|
|
33
|
+
except Exception:
|
|
34
|
+
return f"HTTP {status}"
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@dataclass
|
|
38
|
+
class SubmitAndWaitResult:
|
|
39
|
+
"""Result returned by :meth:`Oleander.submit_spark_job_and_wait`."""
|
|
40
|
+
|
|
41
|
+
run_id: str
|
|
42
|
+
state: str
|
|
43
|
+
run: RunResponse
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class Oleander:
|
|
47
|
+
"""Oleander API client.
|
|
48
|
+
|
|
49
|
+
Mirrors the TypeScript SDK — query lake, list Spark jobs, and launch Spark
|
|
50
|
+
jobs.
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
def __init__(
|
|
54
|
+
self,
|
|
55
|
+
*,
|
|
56
|
+
api_key: Optional[str] = None,
|
|
57
|
+
base_url: Optional[str] = None,
|
|
58
|
+
) -> None:
|
|
59
|
+
opts = OleanderOptions(
|
|
60
|
+
api_key=api_key,
|
|
61
|
+
**({"base_url": base_url} if base_url is not None else {}),
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
resolved_key = (opts.api_key or "").strip() or os.environ.get(
|
|
65
|
+
"OLEANDER_API_KEY", ""
|
|
66
|
+
)
|
|
67
|
+
if not resolved_key:
|
|
68
|
+
raise ValueError(
|
|
69
|
+
"Oleander requires a non-empty api_key (or set OLEANDER_API_KEY)"
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
self._api_key = resolved_key
|
|
73
|
+
self._base_url = (opts.base_url or DEFAULT_BASE_URL).strip().rstrip("/")
|
|
74
|
+
|
|
75
|
+
# ------------------------------------------------------------------
|
|
76
|
+
# Internal helpers
|
|
77
|
+
# ------------------------------------------------------------------
|
|
78
|
+
|
|
79
|
+
def _headers(self, *, content_type: bool = True) -> dict[str, str]:
|
|
80
|
+
h: dict[str, str] = {"Authorization": f"Bearer {self._api_key}"}
|
|
81
|
+
if content_type:
|
|
82
|
+
h["Content-Type"] = "application/json"
|
|
83
|
+
return h
|
|
84
|
+
|
|
85
|
+
async def _request(
|
|
86
|
+
self,
|
|
87
|
+
method: str,
|
|
88
|
+
path: str,
|
|
89
|
+
*,
|
|
90
|
+
json: Optional[dict] = None,
|
|
91
|
+
params: Optional[dict[str, str]] = None,
|
|
92
|
+
content_type: bool = True,
|
|
93
|
+
) -> dict:
|
|
94
|
+
async with httpx.AsyncClient(timeout=30.0) as client:
|
|
95
|
+
resp = await client.request(
|
|
96
|
+
method,
|
|
97
|
+
f"{self._base_url}{path}",
|
|
98
|
+
headers=self._headers(content_type=content_type),
|
|
99
|
+
json=json,
|
|
100
|
+
params=params,
|
|
101
|
+
)
|
|
102
|
+
try:
|
|
103
|
+
body = resp.json()
|
|
104
|
+
except Exception:
|
|
105
|
+
body = {}
|
|
106
|
+
if resp.status_code >= 400:
|
|
107
|
+
raise RuntimeError(_parse_error_body(body, resp.status_code))
|
|
108
|
+
return body
|
|
109
|
+
|
|
110
|
+
# ------------------------------------------------------------------
|
|
111
|
+
# Public API
|
|
112
|
+
# ------------------------------------------------------------------
|
|
113
|
+
|
|
114
|
+
async def query(
|
|
115
|
+
self,
|
|
116
|
+
sql: str,
|
|
117
|
+
options: Optional[QueryOptions] = None,
|
|
118
|
+
) -> LakeQueryResult:
|
|
119
|
+
"""Execute a lake query (mirrors ``oleander query``)."""
|
|
120
|
+
q = (sql or "").strip()
|
|
121
|
+
if not q:
|
|
122
|
+
raise ValueError("Query is required")
|
|
123
|
+
opts = options or QueryOptions()
|
|
124
|
+
|
|
125
|
+
raw = await self._request(
|
|
126
|
+
"POST",
|
|
127
|
+
"/api/v1/warehouse/query",
|
|
128
|
+
json={"query": q, "autoSaveByHash": opts.save},
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
result = LakeQueryResult.model_validate(raw)
|
|
132
|
+
if not result.success and result.error:
|
|
133
|
+
raise RuntimeError(result.details or result.error)
|
|
134
|
+
return result
|
|
135
|
+
|
|
136
|
+
async def list_spark_jobs(
|
|
137
|
+
self,
|
|
138
|
+
options: Optional[ListSparkJobsOptions] = None,
|
|
139
|
+
) -> ListSparkJobsResult:
|
|
140
|
+
"""List spark jobs (mirrors ``oleander spark jobs list``).
|
|
141
|
+
|
|
142
|
+
Uses limit/offset pagination; pages through the API internally.
|
|
143
|
+
"""
|
|
144
|
+
opts = options or ListSparkJobsOptions()
|
|
145
|
+
limit, offset = opts.limit, opts.offset
|
|
146
|
+
|
|
147
|
+
all_scripts: list[str] = []
|
|
148
|
+
continuation_token: Optional[str] = None
|
|
149
|
+
|
|
150
|
+
while True:
|
|
151
|
+
params: Optional[dict[str, str]] = None
|
|
152
|
+
if continuation_token:
|
|
153
|
+
params = {"continuationToken": continuation_token}
|
|
154
|
+
|
|
155
|
+
raw = await self._request(
|
|
156
|
+
"GET",
|
|
157
|
+
"/api/v1/spark/scripts",
|
|
158
|
+
params=params,
|
|
159
|
+
content_type=False,
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
scripts = raw.get("scripts", [])
|
|
163
|
+
all_scripts.extend(scripts)
|
|
164
|
+
continuation_token = raw.get("continuationToken")
|
|
165
|
+
if not continuation_token or len(all_scripts) >= offset + limit:
|
|
166
|
+
break
|
|
167
|
+
|
|
168
|
+
sliced = all_scripts[offset : offset + limit]
|
|
169
|
+
has_more = bool(continuation_token) or len(all_scripts) > offset + limit
|
|
170
|
+
return ListSparkJobsResult(scripts=sliced, has_more=has_more)
|
|
171
|
+
|
|
172
|
+
async def submit_spark_job(
|
|
173
|
+
self,
|
|
174
|
+
options: SparkJobSubmitOptions,
|
|
175
|
+
) -> SparkJobRun:
|
|
176
|
+
"""Submit a spark job (mirrors ``oleander spark jobs submit``).
|
|
177
|
+
|
|
178
|
+
Returns the run ID; use :meth:`get_run` or
|
|
179
|
+
:meth:`submit_spark_job_and_wait` to poll status.
|
|
180
|
+
"""
|
|
181
|
+
raw = await self._request(
|
|
182
|
+
"POST",
|
|
183
|
+
"/api/v1/spark/jobs",
|
|
184
|
+
json={
|
|
185
|
+
"namespace": options.namespace.strip(),
|
|
186
|
+
"name": options.name.strip(),
|
|
187
|
+
"scriptName": options.script_name.strip(),
|
|
188
|
+
"arguments": options.args,
|
|
189
|
+
"driverMachineType": options.driver_machine_type.value,
|
|
190
|
+
"executorMachineType": options.executor_machine_type.value,
|
|
191
|
+
"executorNumbers": options.executor_numbers,
|
|
192
|
+
"jobTags": options.job_tags,
|
|
193
|
+
"runTags": options.run_tags,
|
|
194
|
+
},
|
|
195
|
+
)
|
|
196
|
+
return SparkJobRun.model_validate(raw)
|
|
197
|
+
|
|
198
|
+
async def get_run(self, run_id: str) -> RunResponse:
|
|
199
|
+
"""Get run status (used for polling after submit)."""
|
|
200
|
+
if not run_id or not run_id.strip():
|
|
201
|
+
raise ValueError("run_id is required")
|
|
202
|
+
|
|
203
|
+
raw = await self._request(
|
|
204
|
+
"GET",
|
|
205
|
+
f"/api/v2/runs/{run_id}",
|
|
206
|
+
content_type=False,
|
|
207
|
+
)
|
|
208
|
+
return RunResponse.model_validate(raw)
|
|
209
|
+
|
|
210
|
+
async def submit_spark_job_and_wait(
|
|
211
|
+
self,
|
|
212
|
+
options: SubmitSparkJobAndWaitOptions,
|
|
213
|
+
) -> SubmitAndWaitResult:
|
|
214
|
+
"""Submit a spark job and wait until the run reaches a terminal state
|
|
215
|
+
(COMPLETE, FAIL, ABORT).
|
|
216
|
+
"""
|
|
217
|
+
submit_opts = SparkJobSubmitOptions(
|
|
218
|
+
namespace=options.namespace,
|
|
219
|
+
name=options.name,
|
|
220
|
+
script_name=options.script_name,
|
|
221
|
+
args=options.args,
|
|
222
|
+
driver_machine_type=options.driver_machine_type,
|
|
223
|
+
executor_machine_type=options.executor_machine_type,
|
|
224
|
+
executor_numbers=options.executor_numbers,
|
|
225
|
+
job_tags=options.job_tags,
|
|
226
|
+
run_tags=options.run_tags,
|
|
227
|
+
)
|
|
228
|
+
result = await self.submit_spark_job(submit_opts)
|
|
229
|
+
run_id = result.run_id
|
|
230
|
+
poll_interval_s = options.poll_interval_ms / 1000.0
|
|
231
|
+
timeout_s = options.timeout_ms / 1000.0
|
|
232
|
+
started = time.monotonic()
|
|
233
|
+
|
|
234
|
+
while time.monotonic() - started < timeout_s:
|
|
235
|
+
run = await self.get_run(run_id)
|
|
236
|
+
state = run.state or ""
|
|
237
|
+
if state in TERMINAL_STATES:
|
|
238
|
+
return SubmitAndWaitResult(run_id=run_id, state=state, run=run)
|
|
239
|
+
await asyncio.sleep(poll_interval_s)
|
|
240
|
+
|
|
241
|
+
run = await self.get_run(run_id)
|
|
242
|
+
raise TimeoutError(
|
|
243
|
+
f"Timeout waiting for run {run_id} (state: {run.state or 'unknown'})"
|
|
244
|
+
)
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
"""Pydantic models for the oleander SDK (mirrors schemas.ts)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from enum import Enum
|
|
6
|
+
from typing import Any, Literal, Optional
|
|
7
|
+
|
|
8
|
+
from pydantic import BaseModel, Field
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
# ---------------------------------------------------------------------------
|
|
12
|
+
# Spark machine types
|
|
13
|
+
# ---------------------------------------------------------------------------
|
|
14
|
+
|
|
15
|
+
class SparkMachineType(str, Enum):
|
|
16
|
+
SPARK_1_C = "spark.1.c"
|
|
17
|
+
SPARK_2_C = "spark.2.c"
|
|
18
|
+
SPARK_4_C = "spark.4.c"
|
|
19
|
+
SPARK_8_C = "spark.8.c"
|
|
20
|
+
SPARK_16_C = "spark.16.c"
|
|
21
|
+
SPARK_1_B = "spark.1.b"
|
|
22
|
+
SPARK_2_B = "spark.2.b"
|
|
23
|
+
SPARK_4_B = "spark.4.b"
|
|
24
|
+
SPARK_8_B = "spark.8.b"
|
|
25
|
+
SPARK_16_B = "spark.16.b"
|
|
26
|
+
SPARK_1_M = "spark.1.m"
|
|
27
|
+
SPARK_2_M = "spark.2.m"
|
|
28
|
+
SPARK_4_M = "spark.4.m"
|
|
29
|
+
SPARK_8_M = "spark.8.m"
|
|
30
|
+
SPARK_16_M = "spark.16.m"
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
# ---------------------------------------------------------------------------
|
|
34
|
+
# API error
|
|
35
|
+
# ---------------------------------------------------------------------------
|
|
36
|
+
|
|
37
|
+
class ApiErrorBody(BaseModel):
|
|
38
|
+
error: Optional[str] = None
|
|
39
|
+
details: Optional[str] = None
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
# ---------------------------------------------------------------------------
|
|
43
|
+
# Constructor options
|
|
44
|
+
# ---------------------------------------------------------------------------
|
|
45
|
+
|
|
46
|
+
DEFAULT_BASE_URL = "https://oleander.dev"
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class OleanderOptions(BaseModel):
|
|
50
|
+
api_key: Optional[str] = None
|
|
51
|
+
base_url: str = DEFAULT_BASE_URL
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
# ---------------------------------------------------------------------------
|
|
55
|
+
# Query
|
|
56
|
+
# ---------------------------------------------------------------------------
|
|
57
|
+
|
|
58
|
+
class QueryOptions(BaseModel):
|
|
59
|
+
save: bool = False
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class QueryResultColumns(BaseModel):
|
|
63
|
+
columns: list[str]
|
|
64
|
+
column_types: list[str]
|
|
65
|
+
rows: list[list[Any]]
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class LakeQueryResult(BaseModel):
|
|
69
|
+
success: bool
|
|
70
|
+
results: Optional[QueryResultColumns] = None
|
|
71
|
+
row_count: Optional[int] = None
|
|
72
|
+
execution_time: Optional[str] = None
|
|
73
|
+
saved_table_name: Optional[str] = None
|
|
74
|
+
error: Optional[str] = None
|
|
75
|
+
details: Optional[str] = None
|
|
76
|
+
query: Optional[str] = None
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
# ---------------------------------------------------------------------------
|
|
80
|
+
# Spark job list
|
|
81
|
+
# ---------------------------------------------------------------------------
|
|
82
|
+
|
|
83
|
+
class SparkJobListPage(BaseModel):
|
|
84
|
+
scripts: list[str]
|
|
85
|
+
continuation_token: Optional[str] = Field(None, alias="continuationToken")
|
|
86
|
+
|
|
87
|
+
model_config = {"populate_by_name": True}
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
class ListSparkJobsOptions(BaseModel):
|
|
91
|
+
limit: int = Field(default=20, gt=0)
|
|
92
|
+
offset: int = Field(default=0, ge=0)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
class ListSparkJobsResult(BaseModel):
|
|
96
|
+
scripts: list[str]
|
|
97
|
+
has_more: bool
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
# ---------------------------------------------------------------------------
|
|
101
|
+
# Spark job submit
|
|
102
|
+
# ---------------------------------------------------------------------------
|
|
103
|
+
|
|
104
|
+
class SparkJobSubmitOptions(BaseModel):
|
|
105
|
+
namespace: str = Field(min_length=1)
|
|
106
|
+
name: str = Field(min_length=1)
|
|
107
|
+
script_name: str = Field(min_length=1)
|
|
108
|
+
args: list[str] = Field(default_factory=list)
|
|
109
|
+
driver_machine_type: SparkMachineType = SparkMachineType.SPARK_1_B
|
|
110
|
+
executor_machine_type: SparkMachineType = SparkMachineType.SPARK_1_B
|
|
111
|
+
executor_numbers: int = Field(default=2, ge=1, le=20)
|
|
112
|
+
job_tags: list[str] = Field(default_factory=list)
|
|
113
|
+
run_tags: list[str] = Field(default_factory=list)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
class SubmitSparkJobAndWaitOptions(SparkJobSubmitOptions):
|
|
117
|
+
poll_interval_ms: int = Field(default=10_000, gt=0)
|
|
118
|
+
timeout_ms: int = Field(default=600_000, gt=0)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
class SparkJobRun(BaseModel):
|
|
122
|
+
run_id: str = Field(alias="runId")
|
|
123
|
+
|
|
124
|
+
model_config = {"populate_by_name": True}
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
# ---------------------------------------------------------------------------
|
|
128
|
+
# Run status
|
|
129
|
+
# ---------------------------------------------------------------------------
|
|
130
|
+
|
|
131
|
+
RunState = Literal["COMPLETE", "FAIL", "ABORT"]
|
|
132
|
+
|
|
133
|
+
TERMINAL_STATES: set[str] = {"COMPLETE", "FAIL", "ABORT"}
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
class RunTag(BaseModel):
|
|
137
|
+
key: str
|
|
138
|
+
value: str
|
|
139
|
+
source: Optional[str] = None
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
class RunJobInfo(BaseModel):
|
|
143
|
+
id: str
|
|
144
|
+
name: str
|
|
145
|
+
namespace: str
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
class RunPipelineInfo(BaseModel):
|
|
149
|
+
id: str
|
|
150
|
+
name: str
|
|
151
|
+
namespace: str
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
class RunResponse(BaseModel):
|
|
155
|
+
id: str
|
|
156
|
+
state: Optional[str] = None
|
|
157
|
+
started_at: Optional[str] = None
|
|
158
|
+
queued_at: Optional[str] = None
|
|
159
|
+
scheduled_at: Optional[str] = None
|
|
160
|
+
ended_at: Optional[str] = None
|
|
161
|
+
duration: Optional[float] = None
|
|
162
|
+
error: Optional[Any] = None
|
|
163
|
+
tags: list[RunTag]
|
|
164
|
+
job: RunJobInfo
|
|
165
|
+
pipeline: RunPipelineInfo
|
|
File without changes
|
|
File without changes
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
"""Tests for the oleander Python SDK.
|
|
2
|
+
|
|
3
|
+
Integration tests (list_spark_jobs, query) run only when OLEANDER_API_KEY is set.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import os
|
|
9
|
+
|
|
10
|
+
import pytest
|
|
11
|
+
|
|
12
|
+
from oleander_sdk import (
|
|
13
|
+
Oleander,
|
|
14
|
+
OleanderOptions,
|
|
15
|
+
SparkJobSubmitOptions,
|
|
16
|
+
SparkMachineType,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
# ---------------------------------------------------------------------------
|
|
21
|
+
# Unit tests
|
|
22
|
+
# ---------------------------------------------------------------------------
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class TestOleanderConstructor:
|
|
26
|
+
def test_throws_when_api_key_missing_and_env_unset(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
27
|
+
monkeypatch.delenv("OLEANDER_API_KEY", raising=False)
|
|
28
|
+
with pytest.raises(ValueError, match="Oleander requires a non-empty api_key"):
|
|
29
|
+
Oleander()
|
|
30
|
+
|
|
31
|
+
def test_accepts_api_key(self) -> None:
|
|
32
|
+
oleander = Oleander(api_key="test-key")
|
|
33
|
+
assert oleander is not None
|
|
34
|
+
|
|
35
|
+
def test_uses_env_api_key(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
36
|
+
monkeypatch.setenv("OLEANDER_API_KEY", "env-key")
|
|
37
|
+
oleander = Oleander()
|
|
38
|
+
assert oleander._api_key == "env-key"
|
|
39
|
+
|
|
40
|
+
def test_constructor_api_key_takes_precedence(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
41
|
+
monkeypatch.setenv("OLEANDER_API_KEY", "env-key")
|
|
42
|
+
oleander = Oleander(api_key="explicit-key")
|
|
43
|
+
assert oleander._api_key == "explicit-key"
|
|
44
|
+
|
|
45
|
+
def test_custom_base_url(self) -> None:
|
|
46
|
+
oleander = Oleander(api_key="k", base_url="https://custom.example.com/")
|
|
47
|
+
assert oleander._base_url == "https://custom.example.com"
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class TestOleanderOptions:
|
|
51
|
+
def test_parses_valid_options(self) -> None:
|
|
52
|
+
opts = OleanderOptions(api_key="k", base_url="https://x.com")
|
|
53
|
+
assert opts.api_key == "k"
|
|
54
|
+
assert opts.base_url == "https://x.com"
|
|
55
|
+
|
|
56
|
+
def test_allows_optional_api_key(self) -> None:
|
|
57
|
+
opts = OleanderOptions()
|
|
58
|
+
assert opts.api_key is None
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class TestSubmitOptionsSchema:
|
|
62
|
+
def test_parses_minimal_options_with_defaults(self) -> None:
|
|
63
|
+
opts = SparkJobSubmitOptions(
|
|
64
|
+
namespace="ns",
|
|
65
|
+
name="job",
|
|
66
|
+
script_name="main.py",
|
|
67
|
+
)
|
|
68
|
+
assert opts.namespace == "ns"
|
|
69
|
+
assert opts.name == "job"
|
|
70
|
+
assert opts.script_name == "main.py"
|
|
71
|
+
assert opts.args == []
|
|
72
|
+
assert opts.driver_machine_type == SparkMachineType.SPARK_1_B
|
|
73
|
+
assert opts.executor_numbers == 2
|
|
74
|
+
|
|
75
|
+
def test_rejects_missing_required_fields(self) -> None:
|
|
76
|
+
with pytest.raises(Exception):
|
|
77
|
+
SparkJobSubmitOptions(namespace="ns", name="j") # type: ignore[call-arg]
|
|
78
|
+
with pytest.raises(Exception):
|
|
79
|
+
SparkJobSubmitOptions(namespace="ns", script_name="x.py") # type: ignore[call-arg]
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
# ---------------------------------------------------------------------------
|
|
83
|
+
# Integration tests — only run when OLEANDER_API_KEY is set
|
|
84
|
+
# ---------------------------------------------------------------------------
|
|
85
|
+
|
|
86
|
+
_has_api_key = bool(os.environ.get("OLEANDER_API_KEY"))
|
|
87
|
+
skip_without_key = pytest.mark.skipif(not _has_api_key, reason="OLEANDER_API_KEY not set")
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
@skip_without_key
|
|
91
|
+
class TestListSparkJobsIntegration:
|
|
92
|
+
async def test_returns_shape(self) -> None:
|
|
93
|
+
from oleander_sdk import ListSparkJobsOptions
|
|
94
|
+
|
|
95
|
+
oleander = Oleander()
|
|
96
|
+
result = await oleander.list_spark_jobs(ListSparkJobsOptions(limit=5, offset=0))
|
|
97
|
+
assert isinstance(result.scripts, list)
|
|
98
|
+
assert isinstance(result.has_more, bool)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
@skip_without_key
|
|
102
|
+
class TestQueryIntegration:
|
|
103
|
+
async def test_returns_shape(self) -> None:
|
|
104
|
+
from oleander_sdk import QueryOptions
|
|
105
|
+
|
|
106
|
+
oleander = Oleander()
|
|
107
|
+
result = await oleander.query(
|
|
108
|
+
"SELECT * FROM public.iris_dataset LIMIT 10",
|
|
109
|
+
QueryOptions(save=False),
|
|
110
|
+
)
|
|
111
|
+
assert result.success is True
|
|
112
|
+
assert result.results is not None
|
|
113
|
+
assert result.row_count is not None
|
|
114
|
+
assert result.execution_time is not None
|
|
115
|
+
assert result.saved_table_name is None
|