onedata-lambda-sdk 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- onedata_lambda_sdk-1.0.0/.gitignore +13 -0
- onedata_lambda_sdk-1.0.0/LICENSE.txt +24 -0
- onedata_lambda_sdk-1.0.0/Makefile +61 -0
- onedata_lambda_sdk-1.0.0/PKG-INFO +103 -0
- onedata_lambda_sdk-1.0.0/README.md +61 -0
- onedata_lambda_sdk-1.0.0/docs/_overview.md +241 -0
- onedata_lambda_sdk-1.0.0/docs/guides/file-access.md +171 -0
- onedata_lambda_sdk-1.0.0/docs/guides/local-sdk-vendored-wheel.md +175 -0
- onedata_lambda_sdk-1.0.0/docs/guides/shared-code-uv-workspace.md +213 -0
- onedata_lambda_sdk-1.0.0/docs/guides/single-lambda-repo.md +229 -0
- onedata_lambda_sdk-1.0.0/docs/guides/streaming-logs-and-stats.md +184 -0
- onedata_lambda_sdk-1.0.0/docs/guides/testing-a-lambda.md +177 -0
- onedata_lambda_sdk-1.0.0/docs/guides/writing-a-handler.md +300 -0
- onedata_lambda_sdk-1.0.0/docs/internals/runtime-and-lifecycle.md +210 -0
- onedata_lambda_sdk-1.0.0/onedata_lambda_sdk/__init__.py +60 -0
- onedata_lambda_sdk-1.0.0/onedata_lambda_sdk/_wire.py +111 -0
- onedata_lambda_sdk-1.0.0/onedata_lambda_sdk/job.py +119 -0
- onedata_lambda_sdk-1.0.0/onedata_lambda_sdk/logging.py +99 -0
- onedata_lambda_sdk-1.0.0/onedata_lambda_sdk/oneclient.py +45 -0
- onedata_lambda_sdk-1.0.0/onedata_lambda_sdk/perjob.py +122 -0
- onedata_lambda_sdk-1.0.0/onedata_lambda_sdk/py.typed +0 -0
- onedata_lambda_sdk-1.0.0/onedata_lambda_sdk/runtime.py +388 -0
- onedata_lambda_sdk-1.0.0/onedata_lambda_sdk/stats.py +53 -0
- onedata_lambda_sdk-1.0.0/onedata_lambda_sdk/streaming.py +158 -0
- onedata_lambda_sdk-1.0.0/onedata_lambda_sdk/testing.py +316 -0
- onedata_lambda_sdk-1.0.0/onedata_lambda_sdk/types.py +179 -0
- onedata_lambda_sdk-1.0.0/pyproject.toml +72 -0
- onedata_lambda_sdk-1.0.0/tests/test_job.py +34 -0
- onedata_lambda_sdk-1.0.0/tests/test_logging.py +119 -0
- onedata_lambda_sdk-1.0.0/tests/test_oneclient.py +36 -0
- onedata_lambda_sdk-1.0.0/tests/test_package.py +16 -0
- onedata_lambda_sdk-1.0.0/tests/test_perjob.py +144 -0
- onedata_lambda_sdk-1.0.0/tests/test_runtime.py +134 -0
- onedata_lambda_sdk-1.0.0/tests/test_stats.py +47 -0
- onedata_lambda_sdk-1.0.0/tests/test_streaming.py +186 -0
- onedata_lambda_sdk-1.0.0/tests/test_testing.py +125 -0
- onedata_lambda_sdk-1.0.0/uv.lock +504 -0
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
===========
|
|
3
|
+
|
|
4
|
+
Copyright (C) 2022: Onedata (onedata.org)
|
|
5
|
+
|
|
6
|
+
Permission is hereby granted, free of charge, to any person
|
|
7
|
+
obtaining a copy of this software and associated documentation
|
|
8
|
+
files (the "Software"), to deal in the Software without
|
|
9
|
+
restriction, including without limitation the rights to use, copy,
|
|
10
|
+
modify, merge, publish, distribute, sublicense, and/or sell copies
|
|
11
|
+
of the Software, and to permit persons to whom the Software is
|
|
12
|
+
furnished to do so, subject to the following conditions:
|
|
13
|
+
|
|
14
|
+
The above copyright notice and this permission notice shall be
|
|
15
|
+
included in all copies or substantial portions of the Software.
|
|
16
|
+
|
|
17
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
|
18
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
|
19
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
|
20
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
|
21
|
+
HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
|
22
|
+
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
23
|
+
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
|
24
|
+
OTHER DEALINGS IN THE SOFTWARE.
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
.DEFAULT_GOAL := help
|
|
2
|
+
.PHONY: help sync format format-check static-analysis type-check lint test check build publish clean
|
|
3
|
+
|
|
4
|
+
bold := $(shell tput bold)
|
|
5
|
+
normal := $(shell tput sgr0)
|
|
6
|
+
blue := $(shell tput setaf 4)
|
|
7
|
+
|
|
8
|
+
define print_target
|
|
9
|
+
@echo ""
|
|
10
|
+
@echo "$(blue)$(bold)$@:$(normal)"
|
|
11
|
+
endef
|
|
12
|
+
|
|
13
|
+
# `make help` groups targets by `##@ section` banners and lists each `target: ## description`.
|
|
14
|
+
help:
|
|
15
|
+
@awk 'BEGIN{FS=":.*## "} /^##@ /{printf "\n%s:\n",substr($$0,5)} /^[a-z][a-zA-Z0-9_-]*:.*## /{printf " %-20s %s\n",$$1,$$2}' $(MAKEFILE_LIST)
|
|
16
|
+
|
|
17
|
+
##@ dev
|
|
18
|
+
|
|
19
|
+
sync: ## sync the dev env (uv sync)
|
|
20
|
+
$(call print_target)
|
|
21
|
+
uv sync
|
|
22
|
+
|
|
23
|
+
##@ checks
|
|
24
|
+
|
|
25
|
+
format: ## ruff format + autofix
|
|
26
|
+
$(call print_target)
|
|
27
|
+
uv run ruff format .
|
|
28
|
+
uv run ruff check --fix .
|
|
29
|
+
|
|
30
|
+
format-check: ## ruff format --check
|
|
31
|
+
$(call print_target)
|
|
32
|
+
uv run ruff format --check .
|
|
33
|
+
|
|
34
|
+
static-analysis: ## ruff check
|
|
35
|
+
$(call print_target)
|
|
36
|
+
uv run ruff check .
|
|
37
|
+
|
|
38
|
+
type-check: ## mypy
|
|
39
|
+
$(call print_target)
|
|
40
|
+
uv run mypy onedata_lambda_sdk
|
|
41
|
+
|
|
42
|
+
lint: format-check static-analysis type-check ## format-check + static-analysis + type-check
|
|
43
|
+
@:
|
|
44
|
+
|
|
45
|
+
test: ## pytest (+ coverage, junit for CI)
|
|
46
|
+
$(call print_target)
|
|
47
|
+
uv run pytest --cov=onedata_lambda_sdk --cov-report=term-missing --junitxml=onedata-lambda-sdk-tests-results.xml
|
|
48
|
+
|
|
49
|
+
check: lint test ## lint + test
|
|
50
|
+
@:
|
|
51
|
+
|
|
52
|
+
##@ build & release
|
|
53
|
+
|
|
54
|
+
build: ## uv build (sdist + wheel)
|
|
55
|
+
uv build
|
|
56
|
+
|
|
57
|
+
publish: ## uv publish
|
|
58
|
+
uv publish
|
|
59
|
+
|
|
60
|
+
clean: ## remove build artifacts
|
|
61
|
+
rm -rf build dist *.egg-info
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: onedata-lambda-sdk
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: SDK for writing Onedata Automation lambdas
|
|
5
|
+
Project-URL: Homepage, https://onedata.org
|
|
6
|
+
Author: Bartosz Walkowicz
|
|
7
|
+
License: MIT License
|
|
8
|
+
===========
|
|
9
|
+
|
|
10
|
+
Copyright (C) 2022: Onedata (onedata.org)
|
|
11
|
+
|
|
12
|
+
Permission is hereby granted, free of charge, to any person
|
|
13
|
+
obtaining a copy of this software and associated documentation
|
|
14
|
+
files (the "Software"), to deal in the Software without
|
|
15
|
+
restriction, including without limitation the rights to use, copy,
|
|
16
|
+
modify, merge, publish, distribute, sublicense, and/or sell copies
|
|
17
|
+
of the Software, and to permit persons to whom the Software is
|
|
18
|
+
furnished to do so, subject to the following conditions:
|
|
19
|
+
|
|
20
|
+
The above copyright notice and this permission notice shall be
|
|
21
|
+
included in all copies or substantial portions of the Software.
|
|
22
|
+
|
|
23
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
|
24
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
|
25
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
|
26
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
|
27
|
+
HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
|
28
|
+
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
29
|
+
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
|
30
|
+
OTHER DEALINGS IN THE SOFTWARE.
|
|
31
|
+
License-File: LICENSE.txt
|
|
32
|
+
Keywords: automation,lambda,onedata,openfaas
|
|
33
|
+
Classifier: Intended Audience :: Developers
|
|
34
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
35
|
+
Classifier: Operating System :: OS Independent
|
|
36
|
+
Classifier: Programming Language :: Python :: 3
|
|
37
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
38
|
+
Classifier: Typing :: Typed
|
|
39
|
+
Requires-Python: >=3.12
|
|
40
|
+
Requires-Dist: requests>=2.32
|
|
41
|
+
Description-Content-Type: text/markdown
|
|
42
|
+
|
|
43
|
+
# onedata-lambda-sdk
|
|
44
|
+
|
|
45
|
+
The Python SDK for writing **Onedata Automation lambdas** — the containerized
|
|
46
|
+
operations that run as job executors in automation workflows. You write one
|
|
47
|
+
handler function; the SDK turns the raw OpenFaaS request into typed jobs, runs
|
|
48
|
+
your handler, streams its results, sends the mandatory first heartbeat and keeps
|
|
49
|
+
the batch alive while it streams, and shapes the response the provider expects.
|
|
50
|
+
|
|
51
|
+
```python
|
|
52
|
+
from onedata_lambda_sdk import Job, JobContext, per_job
|
|
53
|
+
|
|
54
|
+
@per_job(max_workers=10)
|
|
55
|
+
def handle(job: Job[dict], ctx: JobContext[dict]) -> dict:
|
|
56
|
+
return {"result": process(job.args)}
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
## Documentation
|
|
60
|
+
|
|
61
|
+
The full authoring documentation lives in [`docs/`](docs/):
|
|
62
|
+
|
|
63
|
+
- **[SDK overview](docs/_overview.md)** — the three-layer model (base image →
|
|
64
|
+
SDK → handler) and how a job batch flows.
|
|
65
|
+
- **Guides** — [writing a handler](docs/guides/writing-a-handler.md),
|
|
66
|
+
[a single-lambda repo](docs/guides/single-lambda-repo.md),
|
|
67
|
+
[several lambdas in a uv workspace](docs/guides/shared-code-uv-workspace.md),
|
|
68
|
+
[developing against a vendored SDK wheel](docs/guides/local-sdk-vendored-wheel.md),
|
|
69
|
+
[testing](docs/guides/testing-a-lambda.md),
|
|
70
|
+
[streaming logs and stats](docs/guides/streaming-logs-and-stats.md), and
|
|
71
|
+
[file access](docs/guides/file-access.md).
|
|
72
|
+
- **[Runtime internals](docs/internals/runtime-and-lifecycle.md)** — for SDK
|
|
73
|
+
maintainers.
|
|
74
|
+
|
|
75
|
+
The platform-side contract this SDK implements — how Oneprovider invokes a
|
|
76
|
+
lambda, the I/O and relay methods, heartbeats — is in the internal developer
|
|
77
|
+
documentation under `docs/automation/internals/lambda/` in
|
|
78
|
+
`onedata-dev-documentation`.
|
|
79
|
+
|
|
80
|
+
## Installation
|
|
81
|
+
|
|
82
|
+
Requires Python 3.12 or newer.
|
|
83
|
+
|
|
84
|
+
```bash
|
|
85
|
+
pip install onedata-lambda-sdk
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
## Development
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
make sync # create the .venv from the lockfile
|
|
92
|
+
make check # lint (ruff format-check + ruff check + mypy) + tests
|
|
93
|
+
make test # pytest with coverage
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
Run `make help` for the full target list.
|
|
97
|
+
|
|
98
|
+
## Compatibility
|
|
99
|
+
|
|
100
|
+
`onedata-lambda-sdk` targets the **v3 lambda model** (`run` / `@per_job` /
|
|
101
|
+
`Job` / `JobContext`) and requires Oneprovider `21.02.5` or newer. A v3 lambda
|
|
102
|
+
builds on the `onedata/lambda-base-slim:v3` base image — see
|
|
103
|
+
[Lambda generations](docs/_overview.md#lambda-generations-v2-and-v3).
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
# onedata-lambda-sdk
|
|
2
|
+
|
|
3
|
+
The Python SDK for writing **Onedata Automation lambdas** — the containerized
|
|
4
|
+
operations that run as job executors in automation workflows. You write one
|
|
5
|
+
handler function; the SDK turns the raw OpenFaaS request into typed jobs, runs
|
|
6
|
+
your handler, streams its results, sends the mandatory first heartbeat and keeps
|
|
7
|
+
the batch alive while it streams, and shapes the response the provider expects.
|
|
8
|
+
|
|
9
|
+
```python
|
|
10
|
+
from onedata_lambda_sdk import Job, JobContext, per_job
|
|
11
|
+
|
|
12
|
+
@per_job(max_workers=10)
|
|
13
|
+
def handle(job: Job[dict], ctx: JobContext[dict]) -> dict:
|
|
14
|
+
return {"result": process(job.args)}
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
## Documentation
|
|
18
|
+
|
|
19
|
+
The full authoring documentation lives in [`docs/`](docs/):
|
|
20
|
+
|
|
21
|
+
- **[SDK overview](docs/_overview.md)** — the three-layer model (base image →
|
|
22
|
+
SDK → handler) and how a job batch flows.
|
|
23
|
+
- **Guides** — [writing a handler](docs/guides/writing-a-handler.md),
|
|
24
|
+
[a single-lambda repo](docs/guides/single-lambda-repo.md),
|
|
25
|
+
[several lambdas in a uv workspace](docs/guides/shared-code-uv-workspace.md),
|
|
26
|
+
[developing against a vendored SDK wheel](docs/guides/local-sdk-vendored-wheel.md),
|
|
27
|
+
[testing](docs/guides/testing-a-lambda.md),
|
|
28
|
+
[streaming logs and stats](docs/guides/streaming-logs-and-stats.md), and
|
|
29
|
+
[file access](docs/guides/file-access.md).
|
|
30
|
+
- **[Runtime internals](docs/internals/runtime-and-lifecycle.md)** — for SDK
|
|
31
|
+
maintainers.
|
|
32
|
+
|
|
33
|
+
The platform-side contract this SDK implements — how Oneprovider invokes a
|
|
34
|
+
lambda, the I/O and relay methods, heartbeats — is in the internal developer
|
|
35
|
+
documentation under `docs/automation/internals/lambda/` in
|
|
36
|
+
`onedata-dev-documentation`.
|
|
37
|
+
|
|
38
|
+
## Installation
|
|
39
|
+
|
|
40
|
+
Requires Python 3.12 or newer.
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
pip install onedata-lambda-sdk
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
## Development
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
make sync # create the .venv from the lockfile
|
|
50
|
+
make check # lint (ruff format-check + ruff check + mypy) + tests
|
|
51
|
+
make test # pytest with coverage
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
Run `make help` for the full target list.
|
|
55
|
+
|
|
56
|
+
## Compatibility
|
|
57
|
+
|
|
58
|
+
`onedata-lambda-sdk` targets the **v3 lambda model** (`run` / `@per_job` /
|
|
59
|
+
`Job` / `JobContext`) and requires Oneprovider `21.02.5` or newer. A v3 lambda
|
|
60
|
+
builds on the `onedata/lambda-base-slim:v3` base image — see
|
|
61
|
+
[Lambda generations](docs/_overview.md#lambda-generations-v2-and-v3).
|
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
---
|
|
2
|
+
audience: integrator
|
|
3
|
+
source_modules:
|
|
4
|
+
- onedata-lambda-sdk/onedata_lambda_sdk/__init__.py
|
|
5
|
+
- onedata-lambda-sdk/onedata_lambda_sdk/runtime.py
|
|
6
|
+
- onedata-lambda-sdk/onedata_lambda_sdk/job.py
|
|
7
|
+
- onedata-lambda-sdk/onedata_lambda_sdk/perjob.py
|
|
8
|
+
- lambda-base/index.py
|
|
9
|
+
- lambda-base/templates/Dockerfile
|
|
10
|
+
source_commits:
|
|
11
|
+
onedata-lambda-sdk: cf25bcd
|
|
12
|
+
lambda-base: 428785a
|
|
13
|
+
---
|
|
14
|
+
|
|
15
|
+
# Onedata Lambda SDK
|
|
16
|
+
|
|
17
|
+
`onedata-lambda-sdk` is the Python SDK for writing **Onedata Automation
|
|
18
|
+
lambdas** — the containerized operations that run as job executors inside
|
|
19
|
+
automation workflows. You write a single handler function; the SDK turns the
|
|
20
|
+
raw OpenFaaS request into typed jobs, runs your handler, streams its results,
|
|
21
|
+
sends the mandatory first heartbeat and keeps the batch alive while it streams,
|
|
22
|
+
and shapes the response the provider expects. Everything between the wire and
|
|
23
|
+
your function is the SDK's job. (A long handler that does *not* stream still
|
|
24
|
+
signals progress itself with `ctx.heartbeat()` — see
|
|
25
|
+
[Writing a handler](guides/writing-a-handler.md).)
|
|
26
|
+
|
|
27
|
+
## The three layers
|
|
28
|
+
|
|
29
|
+
A lambda image is built from three layers, split by how often each changes
|
|
30
|
+
and who owns it:
|
|
31
|
+
|
|
32
|
+
```mermaid
|
|
33
|
+
graph TB
|
|
34
|
+
OP[🖥️ Oneprovider]
|
|
35
|
+
|
|
36
|
+
subgraph Image["🐳 Lambda Docker Image"]
|
|
37
|
+
Base["🐕 lambda-base<br/>watchdog + index.py bootstrap"]
|
|
38
|
+
SDK["🧰 onedata-lambda-sdk SDK<br/>run() + ctx services"]
|
|
39
|
+
Handler["✍️ Handler<br/>your handle() function"]
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
RS["📤 Result Streamer sidecar"]
|
|
43
|
+
OC["📁 Oneclient sidecar"]
|
|
44
|
+
|
|
45
|
+
OP -->|"HTTP job batch"| Base
|
|
46
|
+
Base -->|"run(handler, raw)"| SDK
|
|
47
|
+
SDK -->|"invokes"| Handler
|
|
48
|
+
Handler -->|"uses"| SDK
|
|
49
|
+
Handler -.->|"writes /out/ files"| RS
|
|
50
|
+
Handler -.->|"reads/writes files"| OC
|
|
51
|
+
|
|
52
|
+
classDef ours fill:#4ECDC4,stroke:#0B7285,color:#000
|
|
53
|
+
classDef yours fill:#FFE4B5,stroke:#E8890C,color:#000
|
|
54
|
+
classDef external fill:#A8DADC,stroke:#1864AB,color:#000
|
|
55
|
+
class Base,SDK ours
|
|
56
|
+
class Handler yours
|
|
57
|
+
class OP,RS,OC external
|
|
58
|
+
style Image fill:#EAFBF9,stroke:#0B7285,color:#000
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
- **Base image** (`lambda-base`) — the OS, Python, the OpenFaaS watchdog, and
|
|
62
|
+
a thin, SDK-free bootstrap (`index.py`) that loads your handler and the SDK's
|
|
63
|
+
runtime. It changes rarely and is shared by every lambda. You build on it with
|
|
64
|
+
a one-line `FROM` and never edit its build logic.
|
|
65
|
+
- **SDK** (`onedata-lambda-sdk`, this repo) — the runtime contract
|
|
66
|
+
(`run(handler, raw)`), the handler API (`Job`, `JobContext`), the
|
|
67
|
+
context-vended services (heartbeat, logging, result streaming), the wire
|
|
68
|
+
types, and the test helpers. **Required**: the base loads `run()` from the SDK
|
|
69
|
+
and the build fails if you don't declare it.
|
|
70
|
+
- **Handler** — *your* code: a plain function registered as the single
|
|
71
|
+
`onedata.lambda` entry point. The only layer you write.
|
|
72
|
+
|
|
73
|
+
> [!IMPORTANT]
|
|
74
|
+
> The boundary that matters is **handler vs. everything else**. The base image
|
|
75
|
+
> and the SDK are framework; your handler is the only code you own. Most of this
|
|
76
|
+
> doc set is about that one function and the services it is handed.
|
|
77
|
+
|
|
78
|
+
## Lambda generations: v2 and v3
|
|
79
|
+
|
|
80
|
+
Throughout these docs (and the repo) you will see **v2** and **v3**. They name
|
|
81
|
+
the **lambda generation** — the model a lambda is built against. A generation
|
|
82
|
+
pins two things together:
|
|
83
|
+
|
|
84
|
+
- **The handler shape** — v3 is `run` / `@per_job` / `Job` / `JobContext`
|
|
85
|
+
(this doc set); v2 was a single `handle(request, heartbeat_callback)`.
|
|
86
|
+
- **The base image tag** — a v3 lambda builds on `onedata/lambda-base-slim:v3`.
|
|
87
|
+
|
|
88
|
+
**v3 is the current generation and the only one these docs cover.**
|
|
89
|
+
`onedata-lambda-sdk` is v3-only; v2 lambdas still run, but new lambdas are
|
|
90
|
+
written for v3. See [Compatibility](../README.md#compatibility) for the
|
|
91
|
+
Oneprovider line v3 targets.
|
|
92
|
+
|
|
93
|
+
> [!NOTE]
|
|
94
|
+
> The `v3` in a REST URL such as `/api/v3/oneprovider/data/...` is **unrelated** —
|
|
95
|
+
> that is the Oneprovider data-API version, not the lambda generation.
|
|
96
|
+
|
|
97
|
+
## Why the SDK owns the runtime
|
|
98
|
+
|
|
99
|
+
<sub>source: `onedata-lambda-sdk/onedata_lambda_sdk/runtime.py#run`</sub>
|
|
100
|
+
|
|
101
|
+
In earlier lambda images the per-request orchestration — heartbeats, output
|
|
102
|
+
capture, mount-waiting — lived in the base image's entry point. In
|
|
103
|
+
[v3](#lambda-generations-v2-and-v3) it lives in the SDK's `run()`, and the base
|
|
104
|
+
image's `index.py` is a thin bootstrap that just calls it. Three consequences
|
|
105
|
+
worth knowing:
|
|
106
|
+
|
|
107
|
+
- **The runtime versions with the contract, not the image.** The wire shape,
|
|
108
|
+
the heartbeat protocol, and the result-relay format are SDK concerns; pinning
|
|
109
|
+
the SDK version pins all of them together. A base-image rebuild is no longer
|
|
110
|
+
needed to evolve the contract.
|
|
111
|
+
- **Services are vended by the context, not imported.** Your handler receives a
|
|
112
|
+
`JobContext` and asks it for what it needs — `ctx.logger(name)`,
|
|
113
|
+
`ctx.result_streamer(name)`, `ctx.heartbeat()`. Because the runtime constructs
|
|
114
|
+
these, they can hook into the shared streaming flusher and the heartbeat path,
|
|
115
|
+
and they are swapped for in-memory fakes under test without touching your code.
|
|
116
|
+
- **Failures always become a well-formed response.** Even a broken handler
|
|
117
|
+
import or a malformed request is caught and returned as a top-level
|
|
118
|
+
`{"exception": ...}` envelope, so the provider never sees a naked crash.
|
|
119
|
+
|
|
120
|
+
## How a job batch flows
|
|
121
|
+
|
|
122
|
+
Oneprovider does not call your handler per job — it POSTs a whole **batch** of
|
|
123
|
+
jobs (one shared context plus a list of per-job arguments) and expects a batch
|
|
124
|
+
of results back. The lifecycle of one request:
|
|
125
|
+
|
|
126
|
+
```mermaid
|
|
127
|
+
flowchart TD
|
|
128
|
+
OP[🖥️ Oneprovider] -->|"POST batch · ctx + argsBatch"| Base[🐕 index.py bootstrap]
|
|
129
|
+
Base -->|"run(handler, raw)"| R1
|
|
130
|
+
|
|
131
|
+
subgraph Runtime["🧰 SDK runtime · run()"]
|
|
132
|
+
direction TB
|
|
133
|
+
R1[1 · redirect stdout/stderr] --> R2[2 · await Oneclient mount]
|
|
134
|
+
R2 --> R3[3 · parse request]
|
|
135
|
+
R3 --> R4[4 · first heartbeat]
|
|
136
|
+
R4 --> R5[5 · build Jobs + JobContext]
|
|
137
|
+
R5 --> R6[6 · call handler — your code]
|
|
138
|
+
R6 --> R7[7 · assemble resultsBatch]
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
R4 -.->|"heartbeats"| OP
|
|
142
|
+
R6 -.->|"stream items"| Out[📤 /out streams · sidecar]
|
|
143
|
+
R7 -->|"envelope"| Base
|
|
144
|
+
Base -->|"stdout response"| OP
|
|
145
|
+
|
|
146
|
+
classDef ours fill:#4ECDC4,stroke:#0B7285,color:#000
|
|
147
|
+
classDef yours fill:#FFE4B5,stroke:#E8890C,color:#000
|
|
148
|
+
classDef external fill:#A8DADC,stroke:#1864AB,color:#000
|
|
149
|
+
class Base,R1,R2,R3,R4,R5,R7 ours
|
|
150
|
+
class R6 yours
|
|
151
|
+
class OP,Out external
|
|
152
|
+
style Runtime fill:#EAFBF9,stroke:#0B7285,color:#000
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
1. The base bootstrap reads the request from stdin and calls
|
|
156
|
+
`run(handler, raw)`.
|
|
157
|
+
|
|
158
|
+
<sub>source: `lambda-base/index.py#main`</sub>
|
|
159
|
+
|
|
160
|
+
2. `run()` redirects the process's stdout/stderr to a file so a stray `print`
|
|
161
|
+
cannot corrupt the response.
|
|
162
|
+
3. If the lambda uses a mounted Oneclient, it waits for the mount to become
|
|
163
|
+
ready **before doing anything else** — even before parsing the request. This
|
|
164
|
+
wait can take up to ~120 s, so it comes first; lambdas without a mount skip it.
|
|
165
|
+
4. It parses the request, then delivers the **mandatory first heartbeat** —
|
|
166
|
+
telling Oneprovider the batch actually started — and vends a throttled
|
|
167
|
+
`heartbeat()` for the rest of the run.
|
|
168
|
+
5. It builds the typed `Job` list and the shared `JobContext`, then calls your
|
|
169
|
+
handler.
|
|
170
|
+
|
|
171
|
+
<sub>source: `onedata-lambda-sdk/onedata_lambda_sdk/runtime.py#_build_jobs_and_context`</sub>
|
|
172
|
+
|
|
173
|
+
6. Your handler runs (this is the only step you write).
|
|
174
|
+
7. `run()` wraps the handler's return value in a `{"resultsBatch": [...]}`
|
|
175
|
+
envelope, drains any buffered output streams, and returns it; the base writes
|
|
176
|
+
it to stdout for the provider.
|
|
177
|
+
|
|
178
|
+
## What you write, and what you get
|
|
179
|
+
|
|
180
|
+
Your handler comes in two shapes, both covered in
|
|
181
|
+
[Writing a handler](guides/writing-a-handler.md):
|
|
182
|
+
|
|
183
|
+
```python
|
|
184
|
+
# Batch style — you get the whole batch and return a list the same length.
|
|
185
|
+
def handle(jobs: list[Job[Args]], ctx: JobContext[Config]) -> list[Result]:
|
|
186
|
+
...
|
|
187
|
+
|
|
188
|
+
# Per-job style — the SDK runs the loop; you handle one job at a time.
|
|
189
|
+
@per_job(max_workers=10)
|
|
190
|
+
def handle(job: Job[Args], ctx: JobContext[Config]) -> Result:
|
|
191
|
+
...
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
`Args`, `Config`, and `Result` are your own `TypedDict`s, declared next to the
|
|
195
|
+
handler — see
|
|
196
|
+
[Type your arguments, config, and results](guides/writing-a-handler.md#type-your-arguments-config-and-results).
|
|
197
|
+
`Job` carries that job's typed `args` and its `trace_id`; `JobContext` carries
|
|
198
|
+
the typed task `config`, the read-only identity fields (space, user, provider,
|
|
199
|
+
tokens), and the runtime **services**:
|
|
200
|
+
|
|
201
|
+
| Service | What it does | Guide |
|
|
202
|
+
|---------|--------------|-------|
|
|
203
|
+
| `ctx.heartbeat()` | Signal progress on a long job (throttled by the runtime) | [Writing a handler](guides/writing-a-handler.md) |
|
|
204
|
+
| `ctx.logger(name)` | Emit severity-filtered audit-log entries | [Streaming logs and stats](guides/streaming-logs-and-stats.md) |
|
|
205
|
+
| `ctx.result_streamer(name)` | Append results / time-series measurements to a stream | [Streaming logs and stats](guides/streaming-logs-and-stats.md) |
|
|
206
|
+
|
|
207
|
+
These are the things you would otherwise reimplement in every lambda — a
|
|
208
|
+
heartbeat loop, an output writer, a log formatter. Reach for the service instead
|
|
209
|
+
of hand-rolling it.
|
|
210
|
+
|
|
211
|
+
## What the framework guarantees
|
|
212
|
+
|
|
213
|
+
- **Per-job failure isolation** (with `@per_job`) — one job that raises becomes
|
|
214
|
+
an `AtmException` entry for *that job only*; the rest of the batch still
|
|
215
|
+
returns. See [Writing a handler](guides/writing-a-handler.md#failures).
|
|
216
|
+
- **Expected vs. unexpected errors** — raising `JobException` reports a clean
|
|
217
|
+
message; any other exception is treated as a bug and reported with a full
|
|
218
|
+
traceback.
|
|
219
|
+
- **Output integrity** — stdout/stderr are redirected during the run, so only
|
|
220
|
+
the serialized response reaches the provider.
|
|
221
|
+
- **Liveness** — the first heartbeat is mandatory and automatic; the services
|
|
222
|
+
heartbeat as they flush, so a streaming handler stays alive without extra code.
|
|
223
|
+
- **Result ordering** — results come back in input order even when jobs run in
|
|
224
|
+
parallel.
|
|
225
|
+
|
|
226
|
+
What the SDK does **not** do: decide *how* you reach Onedata files (mounted vs.
|
|
227
|
+
REST — see [File access](guides/file-access.md)), or define the lambda schema
|
|
228
|
+
itself (that is authored alongside the workflow, per the platform docs).
|
|
229
|
+
|
|
230
|
+
## What's next
|
|
231
|
+
|
|
232
|
+
| Goal | Start with |
|
|
233
|
+
|------|------------|
|
|
234
|
+
| Write your first handler | [Writing a handler](guides/writing-a-handler.md) |
|
|
235
|
+
| Ship one lambda from a single repo | [Single-lambda repo](guides/single-lambda-repo.md) |
|
|
236
|
+
| Maintain several lambdas with shared code | [Shared code in a uv workspace](guides/shared-code-uv-workspace.md) |
|
|
237
|
+
| Build a lambda against a locally-changed SDK | [Using a vendored SDK wheel](guides/local-sdk-vendored-wheel.md) |
|
|
238
|
+
| Test a handler without Docker | [Testing a lambda](guides/testing-a-lambda.md) |
|
|
239
|
+
| Emit logs, statistics, or streamed results | [Streaming logs and stats](guides/streaming-logs-and-stats.md) |
|
|
240
|
+
| Read and write Onedata files | [File access](guides/file-access.md) |
|
|
241
|
+
| Understand the runtime internals | [Runtime and lifecycle](internals/runtime-and-lifecycle.md) |
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
---
|
|
2
|
+
audience: integrator
|
|
3
|
+
source_modules:
|
|
4
|
+
- onedata-lambda-sdk/onedata_lambda_sdk/oneclient.py
|
|
5
|
+
- onedata-lambda-sdk/onedata_lambda_sdk/job.py
|
|
6
|
+
- automation-examples/lambdas/calculate-checksum-mounted/src/calculate_checksum_mounted/handler.py
|
|
7
|
+
- automation-examples/lambdas/calculate-checksum-rest/src/calculate_checksum_rest/handler.py
|
|
8
|
+
source_commits:
|
|
9
|
+
onedata-lambda-sdk: cf25bcd
|
|
10
|
+
automation-examples: 1a21949
|
|
11
|
+
---
|
|
12
|
+
|
|
13
|
+
# File access
|
|
14
|
+
|
|
15
|
+
A lambda reaches Onedata files in one of two ways: through a **mounted
|
|
16
|
+
Oneclient** (the space appears as a local filesystem) or through the
|
|
17
|
+
**Oneprovider REST API** (you call HTTP with the job's access token). Which one a
|
|
18
|
+
lambda uses is fixed by its schema; this guide shows how to write the handler for
|
|
19
|
+
each, and how to choose.
|
|
20
|
+
|
|
21
|
+
## When to use this
|
|
22
|
+
|
|
23
|
+
- Your handler reads file content or writes file metadata.
|
|
24
|
+
- You are deciding between a mounted and a REST lambda.
|
|
25
|
+
- You are porting a [v2](../_overview.md#lambda-generations-v2-and-v3) lambda and
|
|
26
|
+
its file access broke (see the third-party-imports note below).
|
|
27
|
+
|
|
28
|
+
> [!NOTE]
|
|
29
|
+
> **Prerequisites.** The [handler model](writing-a-handler.md) — file access uses
|
|
30
|
+
> the job's `args` (an `AtmFile`) and the context's identity fields. The
|
|
31
|
+
> mount-vs-REST switch is `mountOneclient` in the schema's
|
|
32
|
+
> `dockerExecutionOptions`.
|
|
33
|
+
|
|
34
|
+
## Choosing mounted or REST
|
|
35
|
+
|
|
36
|
+
The choice is a property of the lambda, set in its schema — the handler is then
|
|
37
|
+
written for whichever it gets. Use this to decide:
|
|
38
|
+
|
|
39
|
+
| Criterion | Mounted | REST |
|
|
40
|
+
|-----------|---------|------|
|
|
41
|
+
| Large files | preferred | slower |
|
|
42
|
+
| POSIX operations (`xattr`, `stat`) | native | HTTP wrappers |
|
|
43
|
+
| Startup latency | up to ~120 s mount wait | instant |
|
|
44
|
+
| Metadata-only work | overkill | preferred |
|
|
45
|
+
|
|
46
|
+
In short: mounted for heavy content and filesystem semantics, REST for
|
|
47
|
+
metadata-light work and fast startup.
|
|
48
|
+
|
|
49
|
+
## Read through a mounted Oneclient
|
|
50
|
+
|
|
51
|
+
With `mountOneclient: true`, an Oneclient sidecar mounts the space and the SDK
|
|
52
|
+
gives you the path to any file by its id. The runtime waits for the mount to be
|
|
53
|
+
ready before your handler runs, so you can open files directly:
|
|
54
|
+
|
|
55
|
+
```python
|
|
56
|
+
from onedata_lambda_sdk import Job, JobContext, per_job
|
|
57
|
+
from onedata_lambda_sdk.oneclient import mounted_file_path
|
|
58
|
+
|
|
59
|
+
@per_job(max_workers=5)
|
|
60
|
+
def handle(job: Job[JobArgs], ctx: JobContext[TaskConfig]) -> JobResult:
|
|
61
|
+
path = mounted_file_path(job.args["file"]["fileId"])
|
|
62
|
+
with open(path, "rb") as f:
|
|
63
|
+
data = f.read()
|
|
64
|
+
...
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
`mounted_file_path(file_id)` returns `<mount>/.__onedata__file_id__<id>` —
|
|
68
|
+
Oneclient exposes every file at this stable id-based path, so you open a file
|
|
69
|
+
knowing only its id, without resolving where it sits in the tree. Use
|
|
70
|
+
`mount_point()` if you need the mount root itself.
|
|
71
|
+
|
|
72
|
+
> [!TIP]
|
|
73
|
+
> **Source:** `onedata-lambda-sdk/onedata_lambda_sdk/oneclient.py#mounted_file_path`
|
|
74
|
+
> and `#mount_point`. Reading the mount point from this one helper keeps your
|
|
75
|
+
> lambda agreeing with the mount path the runtime waited on, rather than
|
|
76
|
+
> hardcoding `/mnt/onedata`.
|
|
77
|
+
|
|
78
|
+
Because it is a real filesystem, POSIX operations work directly — including
|
|
79
|
+
extended attributes for metadata:
|
|
80
|
+
|
|
81
|
+
```python
|
|
82
|
+
import xattr
|
|
83
|
+
xattr.xattr(path).set("checksum.sha256", value.encode())
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
> [!NOTE]
|
|
87
|
+
> `xattr` is a third-party package, not part of the standard library — so it must
|
|
88
|
+
> be listed in your `pyproject.toml` `dependencies`. The base image installs only
|
|
89
|
+
> what you declare; an undeclared import fails at runtime. This is the same rule —
|
|
90
|
+
> and the same common porting break — as for `requests` in the REST section below.
|
|
91
|
+
|
|
92
|
+
> [!TIP]
|
|
93
|
+
> **Source:** the full mounted example is
|
|
94
|
+
> `automation-examples/lambdas/calculate-checksum-mounted/src/calculate_checksum_mounted/handler.py`.
|
|
95
|
+
|
|
96
|
+
## Read through the REST API
|
|
97
|
+
|
|
98
|
+
With `mountOneclient: false`, there is no filesystem — you call the Oneprovider
|
|
99
|
+
data API using the context's `access_token` and `oneprovider_domain`:
|
|
100
|
+
|
|
101
|
+
```python
|
|
102
|
+
import requests
|
|
103
|
+
|
|
104
|
+
def _file_url(ctx, file_id, subpath):
|
|
105
|
+
return f"https://{ctx.oneprovider_domain}/api/v3/oneprovider/data/{file_id}/{subpath}"
|
|
106
|
+
|
|
107
|
+
response = requests.get(
|
|
108
|
+
_file_url(ctx, file_id, "content"),
|
|
109
|
+
headers={"x-auth-token": ctx.access_token},
|
|
110
|
+
stream=True,
|
|
111
|
+
verify=_verify_ssl(),
|
|
112
|
+
timeout=120,
|
|
113
|
+
)
|
|
114
|
+
response.raise_for_status()
|
|
115
|
+
for chunk in response.iter_content(chunk_size=10 * 1024**2):
|
|
116
|
+
...
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
Writing metadata is a `PUT` to the same base URL:
|
|
120
|
+
|
|
121
|
+
```python
|
|
122
|
+
requests.put(
|
|
123
|
+
_file_url(ctx, file_id, "metadata/xattrs"),
|
|
124
|
+
headers={"x-auth-token": ctx.access_token, "content-type": "application/json"},
|
|
125
|
+
json={"checksum.sha256": value},
|
|
126
|
+
verify=_verify_ssl(),
|
|
127
|
+
timeout=60,
|
|
128
|
+
)
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
Honor the `VERIFY_SSL_CERTIFICATES` environment variable (verification is on
|
|
132
|
+
unless it is set to `"false"`), and read it at call time so tests can flip it:
|
|
133
|
+
|
|
134
|
+
```python
|
|
135
|
+
def _verify_ssl() -> bool:
|
|
136
|
+
return os.environ.get("VERIFY_SSL_CERTIFICATES") != "false"
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
Without a local `stat`, you must request any file attributes you need (`type`,
|
|
140
|
+
`name`, …) through the lambda's `argumentSpecs`, then read them off the
|
|
141
|
+
`AtmFile` — for example `file["type"] != "REG"` to skip non-regular files.
|
|
142
|
+
|
|
143
|
+
> [!WARNING]
|
|
144
|
+
> **Declare every third-party import.** The
|
|
145
|
+
> [v2](../_overview.md#lambda-generations-v2-and-v3) base image shipped `requests`
|
|
146
|
+
> globally, so v2 REST lambdas often imported it without listing it. The v3 base
|
|
147
|
+
> installs only what your `pyproject.toml` `dependencies` declare — `requests`,
|
|
148
|
+
> `xattr`, and anything else outside the standard library. Add the import to
|
|
149
|
+
> `dependencies`, or it fails at runtime. This is the most common v2→v3 porting
|
|
150
|
+
> break.
|
|
151
|
+
|
|
152
|
+
> [!TIP]
|
|
153
|
+
> **Source:** the full REST example is
|
|
154
|
+
> `automation-examples/lambdas/calculate-checksum-rest/src/calculate_checksum_rest/handler.py`,
|
|
155
|
+
> and `ctx.access_token`/`ctx.oneprovider_domain` come from
|
|
156
|
+
> `onedata-lambda-sdk/onedata_lambda_sdk/job.py#JobContext`.
|
|
157
|
+
|
|
158
|
+
## What you get
|
|
159
|
+
|
|
160
|
+
A handler that reads content and writes metadata against Onedata — using the
|
|
161
|
+
mount helpers for filesystem access, or the provider REST API with the job's own
|
|
162
|
+
token — and that fails cleanly per job when a read fails (a transport error
|
|
163
|
+
raised as `JobException` becomes that job's `AtmException` entry).
|
|
164
|
+
|
|
165
|
+
## Next steps
|
|
166
|
+
|
|
167
|
+
- **Report progress while reading large files** —
|
|
168
|
+
[Streaming logs and stats](streaming-logs-and-stats.md).
|
|
169
|
+
- **Test file access without a real mount or provider** —
|
|
170
|
+
[Testing a lambda](testing-a-lambda.md) (point the mount at a temp dir, or mock
|
|
171
|
+
the REST endpoint).
|