onedata-lambda-sdk 1.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. onedata_lambda_sdk-1.0.0/.gitignore +13 -0
  2. onedata_lambda_sdk-1.0.0/LICENSE.txt +24 -0
  3. onedata_lambda_sdk-1.0.0/Makefile +61 -0
  4. onedata_lambda_sdk-1.0.0/PKG-INFO +103 -0
  5. onedata_lambda_sdk-1.0.0/README.md +61 -0
  6. onedata_lambda_sdk-1.0.0/docs/_overview.md +241 -0
  7. onedata_lambda_sdk-1.0.0/docs/guides/file-access.md +171 -0
  8. onedata_lambda_sdk-1.0.0/docs/guides/local-sdk-vendored-wheel.md +175 -0
  9. onedata_lambda_sdk-1.0.0/docs/guides/shared-code-uv-workspace.md +213 -0
  10. onedata_lambda_sdk-1.0.0/docs/guides/single-lambda-repo.md +229 -0
  11. onedata_lambda_sdk-1.0.0/docs/guides/streaming-logs-and-stats.md +184 -0
  12. onedata_lambda_sdk-1.0.0/docs/guides/testing-a-lambda.md +177 -0
  13. onedata_lambda_sdk-1.0.0/docs/guides/writing-a-handler.md +300 -0
  14. onedata_lambda_sdk-1.0.0/docs/internals/runtime-and-lifecycle.md +210 -0
  15. onedata_lambda_sdk-1.0.0/onedata_lambda_sdk/__init__.py +60 -0
  16. onedata_lambda_sdk-1.0.0/onedata_lambda_sdk/_wire.py +111 -0
  17. onedata_lambda_sdk-1.0.0/onedata_lambda_sdk/job.py +119 -0
  18. onedata_lambda_sdk-1.0.0/onedata_lambda_sdk/logging.py +99 -0
  19. onedata_lambda_sdk-1.0.0/onedata_lambda_sdk/oneclient.py +45 -0
  20. onedata_lambda_sdk-1.0.0/onedata_lambda_sdk/perjob.py +122 -0
  21. onedata_lambda_sdk-1.0.0/onedata_lambda_sdk/py.typed +0 -0
  22. onedata_lambda_sdk-1.0.0/onedata_lambda_sdk/runtime.py +388 -0
  23. onedata_lambda_sdk-1.0.0/onedata_lambda_sdk/stats.py +53 -0
  24. onedata_lambda_sdk-1.0.0/onedata_lambda_sdk/streaming.py +158 -0
  25. onedata_lambda_sdk-1.0.0/onedata_lambda_sdk/testing.py +316 -0
  26. onedata_lambda_sdk-1.0.0/onedata_lambda_sdk/types.py +179 -0
  27. onedata_lambda_sdk-1.0.0/pyproject.toml +72 -0
  28. onedata_lambda_sdk-1.0.0/tests/test_job.py +34 -0
  29. onedata_lambda_sdk-1.0.0/tests/test_logging.py +119 -0
  30. onedata_lambda_sdk-1.0.0/tests/test_oneclient.py +36 -0
  31. onedata_lambda_sdk-1.0.0/tests/test_package.py +16 -0
  32. onedata_lambda_sdk-1.0.0/tests/test_perjob.py +144 -0
  33. onedata_lambda_sdk-1.0.0/tests/test_runtime.py +134 -0
  34. onedata_lambda_sdk-1.0.0/tests/test_stats.py +47 -0
  35. onedata_lambda_sdk-1.0.0/tests/test_streaming.py +186 -0
  36. onedata_lambda_sdk-1.0.0/tests/test_testing.py +125 -0
  37. onedata_lambda_sdk-1.0.0/uv.lock +504 -0
@@ -0,0 +1,13 @@
1
+ build
2
+ dist
3
+ onedata_lambda_sdk.egg-info
4
+ __pycache__
5
+ .venv/
6
+ .ruff_cache/
7
+ .mypy_cache/
8
+ .pytest_cache/
9
+ .coverage
10
+ *-tests-results.xml
11
+ .vscode/
12
+ .idea/
13
+ *.iml
@@ -0,0 +1,24 @@
1
+ MIT License
2
+ ===========
3
+
4
+ Copyright (C) 2022: Onedata (onedata.org)
5
+
6
+ Permission is hereby granted, free of charge, to any person
7
+ obtaining a copy of this software and associated documentation
8
+ files (the "Software"), to deal in the Software without
9
+ restriction, including without limitation the rights to use, copy,
10
+ modify, merge, publish, distribute, sublicense, and/or sell copies
11
+ of the Software, and to permit persons to whom the Software is
12
+ furnished to do so, subject to the following conditions:
13
+
14
+ The above copyright notice and this permission notice shall be
15
+ included in all copies or substantial portions of the Software.
16
+
17
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
18
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
19
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
20
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
21
+ HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
22
+ WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
23
+ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
24
+ OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,61 @@
1
+ .DEFAULT_GOAL := help
2
+ .PHONY: help sync format format-check static-analysis type-check lint test check build publish clean
3
+
4
+ bold := $(shell tput bold)
5
+ normal := $(shell tput sgr0)
6
+ blue := $(shell tput setaf 4)
7
+
8
+ define print_target
9
+ @echo ""
10
+ @echo "$(blue)$(bold)$@:$(normal)"
11
+ endef
12
+
13
+ # `make help` groups targets by `##@ section` banners and lists each `target: ## description`.
14
+ help:
15
+ @awk 'BEGIN{FS=":.*## "} /^##@ /{printf "\n%s:\n",substr($$0,5)} /^[a-z][a-zA-Z0-9_-]*:.*## /{printf " %-20s %s\n",$$1,$$2}' $(MAKEFILE_LIST)
16
+
17
+ ##@ dev
18
+
19
+ sync: ## sync the dev env (uv sync)
20
+ $(call print_target)
21
+ uv sync
22
+
23
+ ##@ checks
24
+
25
+ format: ## ruff format + autofix
26
+ $(call print_target)
27
+ uv run ruff format .
28
+ uv run ruff check --fix .
29
+
30
+ format-check: ## ruff format --check
31
+ $(call print_target)
32
+ uv run ruff format --check .
33
+
34
+ static-analysis: ## ruff check
35
+ $(call print_target)
36
+ uv run ruff check .
37
+
38
+ type-check: ## mypy
39
+ $(call print_target)
40
+ uv run mypy onedata_lambda_sdk
41
+
42
+ lint: format-check static-analysis type-check ## format-check + static-analysis + type-check
43
+ @:
44
+
45
+ test: ## pytest (+ coverage, junit for CI)
46
+ $(call print_target)
47
+ uv run pytest --cov=onedata_lambda_sdk --cov-report=term-missing --junitxml=onedata-lambda-sdk-tests-results.xml
48
+
49
+ check: lint test ## lint + test
50
+ @:
51
+
52
+ ##@ build & release
53
+
54
+ build: ## uv build (sdist + wheel)
55
+ uv build
56
+
57
+ publish: ## uv publish
58
+ uv publish
59
+
60
+ clean: ## remove build artifacts
61
+ rm -rf build dist *.egg-info
@@ -0,0 +1,103 @@
1
+ Metadata-Version: 2.5
2
+ Name: onedata-lambda-sdk
3
+ Version: 1.0.0
4
+ Summary: SDK for writing Onedata Automation lambdas
5
+ Project-URL: Homepage, https://onedata.org
6
+ Author: Bartosz Walkowicz
7
+ License: MIT License
8
+ ===========
9
+
10
+ Copyright (C) 2022: Onedata (onedata.org)
11
+
12
+ Permission is hereby granted, free of charge, to any person
13
+ obtaining a copy of this software and associated documentation
14
+ files (the "Software"), to deal in the Software without
15
+ restriction, including without limitation the rights to use, copy,
16
+ modify, merge, publish, distribute, sublicense, and/or sell copies
17
+ of the Software, and to permit persons to whom the Software is
18
+ furnished to do so, subject to the following conditions:
19
+
20
+ The above copyright notice and this permission notice shall be
21
+ included in all copies or substantial portions of the Software.
22
+
23
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
27
+ HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
28
+ WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
29
+ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
30
+ OTHER DEALINGS IN THE SOFTWARE.
31
+ License-File: LICENSE.txt
32
+ Keywords: automation,lambda,onedata,openfaas
33
+ Classifier: Intended Audience :: Developers
34
+ Classifier: License :: OSI Approved :: MIT License
35
+ Classifier: Operating System :: OS Independent
36
+ Classifier: Programming Language :: Python :: 3
37
+ Classifier: Programming Language :: Python :: 3.12
38
+ Classifier: Typing :: Typed
39
+ Requires-Python: >=3.12
40
+ Requires-Dist: requests>=2.32
41
+ Description-Content-Type: text/markdown
42
+
43
+ # onedata-lambda-sdk
44
+
45
+ The Python SDK for writing **Onedata Automation lambdas** — the containerized
46
+ operations that run as job executors in automation workflows. You write one
47
+ handler function; the SDK turns the raw OpenFaaS request into typed jobs, runs
48
+ your handler, streams its results, sends the mandatory first heartbeat and keeps
49
+ the batch alive while it streams, and shapes the response the provider expects.
50
+
51
+ ```python
52
+ from onedata_lambda_sdk import Job, JobContext, per_job
53
+
54
+ @per_job(max_workers=10)
55
+ def handle(job: Job[dict], ctx: JobContext[dict]) -> dict:
56
+ return {"result": process(job.args)}
57
+ ```
58
+
59
+ ## Documentation
60
+
61
+ The full authoring documentation lives in [`docs/`](docs/):
62
+
63
+ - **[SDK overview](docs/_overview.md)** — the three-layer model (base image →
64
+ SDK → handler) and how a job batch flows.
65
+ - **Guides** — [writing a handler](docs/guides/writing-a-handler.md),
66
+ [a single-lambda repo](docs/guides/single-lambda-repo.md),
67
+ [several lambdas in a uv workspace](docs/guides/shared-code-uv-workspace.md),
68
+ [developing against a vendored SDK wheel](docs/guides/local-sdk-vendored-wheel.md),
69
+ [testing](docs/guides/testing-a-lambda.md),
70
+ [streaming logs and stats](docs/guides/streaming-logs-and-stats.md), and
71
+ [file access](docs/guides/file-access.md).
72
+ - **[Runtime internals](docs/internals/runtime-and-lifecycle.md)** — for SDK
73
+ maintainers.
74
+
75
+ The platform-side contract this SDK implements — how Oneprovider invokes a
76
+ lambda, the I/O and relay methods, heartbeats — is in the internal developer
77
+ documentation under `docs/automation/internals/lambda/` in
78
+ `onedata-dev-documentation`.
79
+
80
+ ## Installation
81
+
82
+ Requires Python 3.12 or newer.
83
+
84
+ ```bash
85
+ pip install onedata-lambda-sdk
86
+ ```
87
+
88
+ ## Development
89
+
90
+ ```bash
91
+ make sync # create the .venv from the lockfile
92
+ make check # lint (ruff format-check + ruff check + mypy) + tests
93
+ make test # pytest with coverage
94
+ ```
95
+
96
+ Run `make help` for the full target list.
97
+
98
+ ## Compatibility
99
+
100
+ `onedata-lambda-sdk` targets the **v3 lambda model** (`run` / `@per_job` /
101
+ `Job` / `JobContext`) and requires Oneprovider `21.02.5` or newer. A v3 lambda
102
+ builds on the `onedata/lambda-base-slim:v3` base image — see
103
+ [Lambda generations](docs/_overview.md#lambda-generations-v2-and-v3).
@@ -0,0 +1,61 @@
1
+ # onedata-lambda-sdk
2
+
3
+ The Python SDK for writing **Onedata Automation lambdas** — the containerized
4
+ operations that run as job executors in automation workflows. You write one
5
+ handler function; the SDK turns the raw OpenFaaS request into typed jobs, runs
6
+ your handler, streams its results, sends the mandatory first heartbeat and keeps
7
+ the batch alive while it streams, and shapes the response the provider expects.
8
+
9
+ ```python
10
+ from onedata_lambda_sdk import Job, JobContext, per_job
11
+
12
+ @per_job(max_workers=10)
13
+ def handle(job: Job[dict], ctx: JobContext[dict]) -> dict:
14
+ return {"result": process(job.args)}
15
+ ```
16
+
17
+ ## Documentation
18
+
19
+ The full authoring documentation lives in [`docs/`](docs/):
20
+
21
+ - **[SDK overview](docs/_overview.md)** — the three-layer model (base image →
22
+ SDK → handler) and how a job batch flows.
23
+ - **Guides** — [writing a handler](docs/guides/writing-a-handler.md),
24
+ [a single-lambda repo](docs/guides/single-lambda-repo.md),
25
+ [several lambdas in a uv workspace](docs/guides/shared-code-uv-workspace.md),
26
+ [developing against a vendored SDK wheel](docs/guides/local-sdk-vendored-wheel.md),
27
+ [testing](docs/guides/testing-a-lambda.md),
28
+ [streaming logs and stats](docs/guides/streaming-logs-and-stats.md), and
29
+ [file access](docs/guides/file-access.md).
30
+ - **[Runtime internals](docs/internals/runtime-and-lifecycle.md)** — for SDK
31
+ maintainers.
32
+
33
+ The platform-side contract this SDK implements — how Oneprovider invokes a
34
+ lambda, the I/O and relay methods, heartbeats — is in the internal developer
35
+ documentation under `docs/automation/internals/lambda/` in
36
+ `onedata-dev-documentation`.
37
+
38
+ ## Installation
39
+
40
+ Requires Python 3.12 or newer.
41
+
42
+ ```bash
43
+ pip install onedata-lambda-sdk
44
+ ```
45
+
46
+ ## Development
47
+
48
+ ```bash
49
+ make sync # create the .venv from the lockfile
50
+ make check # lint (ruff format-check + ruff check + mypy) + tests
51
+ make test # pytest with coverage
52
+ ```
53
+
54
+ Run `make help` for the full target list.
55
+
56
+ ## Compatibility
57
+
58
+ `onedata-lambda-sdk` targets the **v3 lambda model** (`run` / `@per_job` /
59
+ `Job` / `JobContext`) and requires Oneprovider `21.02.5` or newer. A v3 lambda
60
+ builds on the `onedata/lambda-base-slim:v3` base image — see
61
+ [Lambda generations](docs/_overview.md#lambda-generations-v2-and-v3).
@@ -0,0 +1,241 @@
1
+ ---
2
+ audience: integrator
3
+ source_modules:
4
+ - onedata-lambda-sdk/onedata_lambda_sdk/__init__.py
5
+ - onedata-lambda-sdk/onedata_lambda_sdk/runtime.py
6
+ - onedata-lambda-sdk/onedata_lambda_sdk/job.py
7
+ - onedata-lambda-sdk/onedata_lambda_sdk/perjob.py
8
+ - lambda-base/index.py
9
+ - lambda-base/templates/Dockerfile
10
+ source_commits:
11
+ onedata-lambda-sdk: cf25bcd
12
+ lambda-base: 428785a
13
+ ---
14
+
15
+ # Onedata Lambda SDK
16
+
17
+ `onedata-lambda-sdk` is the Python SDK for writing **Onedata Automation
18
+ lambdas** — the containerized operations that run as job executors inside
19
+ automation workflows. You write a single handler function; the SDK turns the
20
+ raw OpenFaaS request into typed jobs, runs your handler, streams its results,
21
+ sends the mandatory first heartbeat and keeps the batch alive while it streams,
22
+ and shapes the response the provider expects. Everything between the wire and
23
+ your function is the SDK's job. (A long handler that does *not* stream still
24
+ signals progress itself with `ctx.heartbeat()` — see
25
+ [Writing a handler](guides/writing-a-handler.md).)
26
+
27
+ ## The three layers
28
+
29
+ A lambda image is built from three layers, split by how often each changes
30
+ and who owns it:
31
+
32
+ ```mermaid
33
+ graph TB
34
+ OP[🖥️ Oneprovider]
35
+
36
+ subgraph Image["🐳 Lambda Docker Image"]
37
+ Base["🐕 lambda-base<br/>watchdog + index.py bootstrap"]
38
+ SDK["🧰 onedata-lambda-sdk SDK<br/>run() + ctx services"]
39
+ Handler["✍️ Handler<br/>your handle() function"]
40
+ end
41
+
42
+ RS["📤 Result Streamer sidecar"]
43
+ OC["📁 Oneclient sidecar"]
44
+
45
+ OP -->|"HTTP job batch"| Base
46
+ Base -->|"run(handler, raw)"| SDK
47
+ SDK -->|"invokes"| Handler
48
+ Handler -->|"uses"| SDK
49
+ Handler -.->|"writes /out/ files"| RS
50
+ Handler -.->|"reads/writes files"| OC
51
+
52
+ classDef ours fill:#4ECDC4,stroke:#0B7285,color:#000
53
+ classDef yours fill:#FFE4B5,stroke:#E8890C,color:#000
54
+ classDef external fill:#A8DADC,stroke:#1864AB,color:#000
55
+ class Base,SDK ours
56
+ class Handler yours
57
+ class OP,RS,OC external
58
+ style Image fill:#EAFBF9,stroke:#0B7285,color:#000
59
+ ```
60
+
61
+ - **Base image** (`lambda-base`) — the OS, Python, the OpenFaaS watchdog, and
62
+ a thin, SDK-free bootstrap (`index.py`) that loads your handler and the SDK's
63
+ runtime. It changes rarely and is shared by every lambda. You build on it with
64
+ a one-line `FROM` and never edit its build logic.
65
+ - **SDK** (`onedata-lambda-sdk`, this repo) — the runtime contract
66
+ (`run(handler, raw)`), the handler API (`Job`, `JobContext`), the
67
+ context-vended services (heartbeat, logging, result streaming), the wire
68
+ types, and the test helpers. **Required**: the base loads `run()` from the SDK
69
+ and the build fails if you don't declare it.
70
+ - **Handler** — *your* code: a plain function registered as the single
71
+ `onedata.lambda` entry point. The only layer you write.
72
+
73
+ > [!IMPORTANT]
74
+ > The boundary that matters is **handler vs. everything else**. The base image
75
+ > and the SDK are framework; your handler is the only code you own. Most of this
76
+ > doc set is about that one function and the services it is handed.
77
+
78
+ ## Lambda generations: v2 and v3
79
+
80
+ Throughout these docs (and the repo) you will see **v2** and **v3**. They name
81
+ the **lambda generation** — the model a lambda is built against. A generation
82
+ pins two things together:
83
+
84
+ - **The handler shape** — v3 is `run` / `@per_job` / `Job` / `JobContext`
85
+ (this doc set); v2 was a single `handle(request, heartbeat_callback)`.
86
+ - **The base image tag** — a v3 lambda builds on `onedata/lambda-base-slim:v3`.
87
+
88
+ **v3 is the current generation and the only one these docs cover.**
89
+ `onedata-lambda-sdk` is v3-only; v2 lambdas still run, but new lambdas are
90
+ written for v3. See [Compatibility](../README.md#compatibility) for the
91
+ Oneprovider line v3 targets.
92
+
93
+ > [!NOTE]
94
+ > The `v3` in a REST URL such as `/api/v3/oneprovider/data/...` is **unrelated** —
95
+ > that is the Oneprovider data-API version, not the lambda generation.
96
+
97
+ ## Why the SDK owns the runtime
98
+
99
+ <sub>source: `onedata-lambda-sdk/onedata_lambda_sdk/runtime.py#run`</sub>
100
+
101
+ In earlier lambda images the per-request orchestration — heartbeats, output
102
+ capture, mount-waiting — lived in the base image's entry point. In
103
+ [v3](#lambda-generations-v2-and-v3) it lives in the SDK's `run()`, and the base
104
+ image's `index.py` is a thin bootstrap that just calls it. Three consequences
105
+ worth knowing:
106
+
107
+ - **The runtime versions with the contract, not the image.** The wire shape,
108
+ the heartbeat protocol, and the result-relay format are SDK concerns; pinning
109
+ the SDK version pins all of them together. A base-image rebuild is no longer
110
+ needed to evolve the contract.
111
+ - **Services are vended by the context, not imported.** Your handler receives a
112
+ `JobContext` and asks it for what it needs — `ctx.logger(name)`,
113
+ `ctx.result_streamer(name)`, `ctx.heartbeat()`. Because the runtime constructs
114
+ these, they can hook into the shared streaming flusher and the heartbeat path,
115
+ and they are swapped for in-memory fakes under test without touching your code.
116
+ - **Failures always become a well-formed response.** Even a broken handler
117
+ import or a malformed request is caught and returned as a top-level
118
+ `{"exception": ...}` envelope, so the provider never sees a naked crash.
119
+
120
+ ## How a job batch flows
121
+
122
+ Oneprovider does not call your handler per job — it POSTs a whole **batch** of
123
+ jobs (one shared context plus a list of per-job arguments) and expects a batch
124
+ of results back. The lifecycle of one request:
125
+
126
+ ```mermaid
127
+ flowchart TD
128
+ OP[🖥️ Oneprovider] -->|"POST batch · ctx + argsBatch"| Base[🐕 index.py bootstrap]
129
+ Base -->|"run(handler, raw)"| R1
130
+
131
+ subgraph Runtime["🧰 SDK runtime · run()"]
132
+ direction TB
133
+ R1[1 · redirect stdout/stderr] --> R2[2 · await Oneclient mount]
134
+ R2 --> R3[3 · parse request]
135
+ R3 --> R4[4 · first heartbeat]
136
+ R4 --> R5[5 · build Jobs + JobContext]
137
+ R5 --> R6[6 · call handler — your code]
138
+ R6 --> R7[7 · assemble resultsBatch]
139
+ end
140
+
141
+ R4 -.->|"heartbeats"| OP
142
+ R6 -.->|"stream items"| Out[📤 /out streams · sidecar]
143
+ R7 -->|"envelope"| Base
144
+ Base -->|"stdout response"| OP
145
+
146
+ classDef ours fill:#4ECDC4,stroke:#0B7285,color:#000
147
+ classDef yours fill:#FFE4B5,stroke:#E8890C,color:#000
148
+ classDef external fill:#A8DADC,stroke:#1864AB,color:#000
149
+ class Base,R1,R2,R3,R4,R5,R7 ours
150
+ class R6 yours
151
+ class OP,Out external
152
+ style Runtime fill:#EAFBF9,stroke:#0B7285,color:#000
153
+ ```
154
+
155
+ 1. The base bootstrap reads the request from stdin and calls
156
+ `run(handler, raw)`.
157
+
158
+ <sub>source: `lambda-base/index.py#main`</sub>
159
+
160
+ 2. `run()` redirects the process's stdout/stderr to a file so a stray `print`
161
+ cannot corrupt the response.
162
+ 3. If the lambda uses a mounted Oneclient, it waits for the mount to become
163
+ ready **before doing anything else** — even before parsing the request. This
164
+ wait can take up to ~120 s, so it comes first; lambdas without a mount skip it.
165
+ 4. It parses the request, then delivers the **mandatory first heartbeat** —
166
+ telling Oneprovider the batch actually started — and vends a throttled
167
+ `heartbeat()` for the rest of the run.
168
+ 5. It builds the typed `Job` list and the shared `JobContext`, then calls your
169
+ handler.
170
+
171
+ <sub>source: `onedata-lambda-sdk/onedata_lambda_sdk/runtime.py#_build_jobs_and_context`</sub>
172
+
173
+ 6. Your handler runs (this is the only step you write).
174
+ 7. `run()` wraps the handler's return value in a `{"resultsBatch": [...]}`
175
+ envelope, drains any buffered output streams, and returns it; the base writes
176
+ it to stdout for the provider.
177
+
178
+ ## What you write, and what you get
179
+
180
+ Your handler comes in two shapes, both covered in
181
+ [Writing a handler](guides/writing-a-handler.md):
182
+
183
+ ```python
184
+ # Batch style — you get the whole batch and return a list the same length.
185
+ def handle(jobs: list[Job[Args]], ctx: JobContext[Config]) -> list[Result]:
186
+ ...
187
+
188
+ # Per-job style — the SDK runs the loop; you handle one job at a time.
189
+ @per_job(max_workers=10)
190
+ def handle(job: Job[Args], ctx: JobContext[Config]) -> Result:
191
+ ...
192
+ ```
193
+
194
+ `Args`, `Config`, and `Result` are your own `TypedDict`s, declared next to the
195
+ handler — see
196
+ [Type your arguments, config, and results](guides/writing-a-handler.md#type-your-arguments-config-and-results).
197
+ `Job` carries that job's typed `args` and its `trace_id`; `JobContext` carries
198
+ the typed task `config`, the read-only identity fields (space, user, provider,
199
+ tokens), and the runtime **services**:
200
+
201
+ | Service | What it does | Guide |
202
+ |---------|--------------|-------|
203
+ | `ctx.heartbeat()` | Signal progress on a long job (throttled by the runtime) | [Writing a handler](guides/writing-a-handler.md) |
204
+ | `ctx.logger(name)` | Emit severity-filtered audit-log entries | [Streaming logs and stats](guides/streaming-logs-and-stats.md) |
205
+ | `ctx.result_streamer(name)` | Append results / time-series measurements to a stream | [Streaming logs and stats](guides/streaming-logs-and-stats.md) |
206
+
207
+ These are the things you would otherwise reimplement in every lambda — a
208
+ heartbeat loop, an output writer, a log formatter. Reach for the service instead
209
+ of hand-rolling it.
210
+
211
+ ## What the framework guarantees
212
+
213
+ - **Per-job failure isolation** (with `@per_job`) — one job that raises becomes
214
+ an `AtmException` entry for *that job only*; the rest of the batch still
215
+ returns. See [Writing a handler](guides/writing-a-handler.md#failures).
216
+ - **Expected vs. unexpected errors** — raising `JobException` reports a clean
217
+ message; any other exception is treated as a bug and reported with a full
218
+ traceback.
219
+ - **Output integrity** — stdout/stderr are redirected during the run, so only
220
+ the serialized response reaches the provider.
221
+ - **Liveness** — the first heartbeat is mandatory and automatic; the services
222
+ heartbeat as they flush, so a streaming handler stays alive without extra code.
223
+ - **Result ordering** — results come back in input order even when jobs run in
224
+ parallel.
225
+
226
+ What the SDK does **not** do: decide *how* you reach Onedata files (mounted vs.
227
+ REST — see [File access](guides/file-access.md)), or define the lambda schema
228
+ itself (that is authored alongside the workflow, per the platform docs).
229
+
230
+ ## What's next
231
+
232
+ | Goal | Start with |
233
+ |------|------------|
234
+ | Write your first handler | [Writing a handler](guides/writing-a-handler.md) |
235
+ | Ship one lambda from a single repo | [Single-lambda repo](guides/single-lambda-repo.md) |
236
+ | Maintain several lambdas with shared code | [Shared code in a uv workspace](guides/shared-code-uv-workspace.md) |
237
+ | Build a lambda against a locally-changed SDK | [Using a vendored SDK wheel](guides/local-sdk-vendored-wheel.md) |
238
+ | Test a handler without Docker | [Testing a lambda](guides/testing-a-lambda.md) |
239
+ | Emit logs, statistics, or streamed results | [Streaming logs and stats](guides/streaming-logs-and-stats.md) |
240
+ | Read and write Onedata files | [File access](guides/file-access.md) |
241
+ | Understand the runtime internals | [Runtime and lifecycle](internals/runtime-and-lifecycle.md) |
@@ -0,0 +1,171 @@
1
+ ---
2
+ audience: integrator
3
+ source_modules:
4
+ - onedata-lambda-sdk/onedata_lambda_sdk/oneclient.py
5
+ - onedata-lambda-sdk/onedata_lambda_sdk/job.py
6
+ - automation-examples/lambdas/calculate-checksum-mounted/src/calculate_checksum_mounted/handler.py
7
+ - automation-examples/lambdas/calculate-checksum-rest/src/calculate_checksum_rest/handler.py
8
+ source_commits:
9
+ onedata-lambda-sdk: cf25bcd
10
+ automation-examples: 1a21949
11
+ ---
12
+
13
+ # File access
14
+
15
+ A lambda reaches Onedata files in one of two ways: through a **mounted
16
+ Oneclient** (the space appears as a local filesystem) or through the
17
+ **Oneprovider REST API** (you call HTTP with the job's access token). Which one a
18
+ lambda uses is fixed by its schema; this guide shows how to write the handler for
19
+ each, and how to choose.
20
+
21
+ ## When to use this
22
+
23
+ - Your handler reads file content or writes file metadata.
24
+ - You are deciding between a mounted and a REST lambda.
25
+ - You are porting a [v2](../_overview.md#lambda-generations-v2-and-v3) lambda and
26
+ its file access broke (see the third-party-imports note below).
27
+
28
+ > [!NOTE]
29
+ > **Prerequisites.** The [handler model](writing-a-handler.md) — file access uses
30
+ > the job's `args` (an `AtmFile`) and the context's identity fields. The
31
+ > mount-vs-REST switch is `mountOneclient` in the schema's
32
+ > `dockerExecutionOptions`.
33
+
34
+ ## Choosing mounted or REST
35
+
36
+ The choice is a property of the lambda, set in its schema — the handler is then
37
+ written for whichever it gets. Use this to decide:
38
+
39
+ | Criterion | Mounted | REST |
40
+ |-----------|---------|------|
41
+ | Large files | preferred | slower |
42
+ | POSIX operations (`xattr`, `stat`) | native | HTTP wrappers |
43
+ | Startup latency | up to ~120 s mount wait | instant |
44
+ | Metadata-only work | overkill | preferred |
45
+
46
+ In short: mounted for heavy content and filesystem semantics, REST for
47
+ metadata-light work and fast startup.
48
+
49
+ ## Read through a mounted Oneclient
50
+
51
+ With `mountOneclient: true`, an Oneclient sidecar mounts the space and the SDK
52
+ gives you the path to any file by its id. The runtime waits for the mount to be
53
+ ready before your handler runs, so you can open files directly:
54
+
55
+ ```python
56
+ from onedata_lambda_sdk import Job, JobContext, per_job
57
+ from onedata_lambda_sdk.oneclient import mounted_file_path
58
+
59
+ @per_job(max_workers=5)
60
+ def handle(job: Job[JobArgs], ctx: JobContext[TaskConfig]) -> JobResult:
61
+ path = mounted_file_path(job.args["file"]["fileId"])
62
+ with open(path, "rb") as f:
63
+ data = f.read()
64
+ ...
65
+ ```
66
+
67
+ `mounted_file_path(file_id)` returns `<mount>/.__onedata__file_id__<id>` —
68
+ Oneclient exposes every file at this stable id-based path, so you open a file
69
+ knowing only its id, without resolving where it sits in the tree. Use
70
+ `mount_point()` if you need the mount root itself.
71
+
72
+ > [!TIP]
73
+ > **Source:** `onedata-lambda-sdk/onedata_lambda_sdk/oneclient.py#mounted_file_path`
74
+ > and `#mount_point`. Reading the mount point from this one helper keeps your
75
+ > lambda agreeing with the mount path the runtime waited on, rather than
76
+ > hardcoding `/mnt/onedata`.
77
+
78
+ Because it is a real filesystem, POSIX operations work directly — including
79
+ extended attributes for metadata:
80
+
81
+ ```python
82
+ import xattr
83
+ xattr.xattr(path).set("checksum.sha256", value.encode())
84
+ ```
85
+
86
+ > [!NOTE]
87
+ > `xattr` is a third-party package, not part of the standard library — so it must
88
+ > be listed in your `pyproject.toml` `dependencies`. The base image installs only
89
+ > what you declare; an undeclared import fails at runtime. This is the same rule —
90
+ > and the same common porting break — as for `requests` in the REST section below.
91
+
92
+ > [!TIP]
93
+ > **Source:** the full mounted example is
94
+ > `automation-examples/lambdas/calculate-checksum-mounted/src/calculate_checksum_mounted/handler.py`.
95
+
96
+ ## Read through the REST API
97
+
98
+ With `mountOneclient: false`, there is no filesystem — you call the Oneprovider
99
+ data API using the context's `access_token` and `oneprovider_domain`:
100
+
101
+ ```python
102
+ import requests
103
+
104
+ def _file_url(ctx, file_id, subpath):
105
+ return f"https://{ctx.oneprovider_domain}/api/v3/oneprovider/data/{file_id}/{subpath}"
106
+
107
+ response = requests.get(
108
+ _file_url(ctx, file_id, "content"),
109
+ headers={"x-auth-token": ctx.access_token},
110
+ stream=True,
111
+ verify=_verify_ssl(),
112
+ timeout=120,
113
+ )
114
+ response.raise_for_status()
115
+ for chunk in response.iter_content(chunk_size=10 * 1024**2):
116
+ ...
117
+ ```
118
+
119
+ Writing metadata is a `PUT` to the same base URL:
120
+
121
+ ```python
122
+ requests.put(
123
+ _file_url(ctx, file_id, "metadata/xattrs"),
124
+ headers={"x-auth-token": ctx.access_token, "content-type": "application/json"},
125
+ json={"checksum.sha256": value},
126
+ verify=_verify_ssl(),
127
+ timeout=60,
128
+ )
129
+ ```
130
+
131
+ Honor the `VERIFY_SSL_CERTIFICATES` environment variable (verification is on
132
+ unless it is set to `"false"`), and read it at call time so tests can flip it:
133
+
134
+ ```python
135
+ def _verify_ssl() -> bool:
136
+ return os.environ.get("VERIFY_SSL_CERTIFICATES") != "false"
137
+ ```
138
+
139
+ Without a local `stat`, you must request any file attributes you need (`type`,
140
+ `name`, …) through the lambda's `argumentSpecs`, then read them off the
141
+ `AtmFile` — for example `file["type"] != "REG"` to skip non-regular files.
142
+
143
+ > [!WARNING]
144
+ > **Declare every third-party import.** The
145
+ > [v2](../_overview.md#lambda-generations-v2-and-v3) base image shipped `requests`
146
+ > globally, so v2 REST lambdas often imported it without listing it. The v3 base
147
+ > installs only what your `pyproject.toml` `dependencies` declare — `requests`,
148
+ > `xattr`, and anything else outside the standard library. Add the import to
149
+ > `dependencies`, or it fails at runtime. This is the most common v2→v3 porting
150
+ > break.
151
+
152
+ > [!TIP]
153
+ > **Source:** the full REST example is
154
+ > `automation-examples/lambdas/calculate-checksum-rest/src/calculate_checksum_rest/handler.py`,
155
+ > and `ctx.access_token`/`ctx.oneprovider_domain` come from
156
+ > `onedata-lambda-sdk/onedata_lambda_sdk/job.py#JobContext`.
157
+
158
+ ## What you get
159
+
160
+ A handler that reads content and writes metadata against Onedata — using the
161
+ mount helpers for filesystem access, or the provider REST API with the job's own
162
+ token — and that fails cleanly per job when a read fails (a transport error
163
+ raised as `JobException` becomes that job's `AtmException` entry).
164
+
165
+ ## Next steps
166
+
167
+ - **Report progress while reading large files** —
168
+ [Streaming logs and stats](streaming-logs-and-stats.md).
169
+ - **Test file access without a real mount or provider** —
170
+ [Testing a lambda](testing-a-lambda.md) (point the mount at a temp dir, or mock
171
+ the REST endpoint).