onedata-lambda-sdk 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- onedata_lambda_sdk/__init__.py +60 -0
- onedata_lambda_sdk/_wire.py +111 -0
- onedata_lambda_sdk/job.py +119 -0
- onedata_lambda_sdk/logging.py +99 -0
- onedata_lambda_sdk/oneclient.py +45 -0
- onedata_lambda_sdk/perjob.py +122 -0
- onedata_lambda_sdk/py.typed +0 -0
- onedata_lambda_sdk/runtime.py +388 -0
- onedata_lambda_sdk/stats.py +53 -0
- onedata_lambda_sdk/streaming.py +158 -0
- onedata_lambda_sdk/testing.py +316 -0
- onedata_lambda_sdk/types.py +179 -0
- onedata_lambda_sdk-1.0.0.dist-info/METADATA +103 -0
- onedata_lambda_sdk-1.0.0.dist-info/RECORD +16 -0
- onedata_lambda_sdk-1.0.0.dist-info/WHEEL +4 -0
- onedata_lambda_sdk-1.0.0.dist-info/licenses/LICENSE.txt +24 -0
|
@@ -0,0 +1,316 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Helpers for unit-testing lambda handlers without Docker, network, or the runtime.
|
|
3
|
+
|
|
4
|
+
Build a synthetic context and jobs, call your handler directly, then assert on both its
|
|
5
|
+
return value and its side effects -- heartbeats, logs and streamed items, captured in
|
|
6
|
+
memory per name:
|
|
7
|
+
|
|
8
|
+
from onedata_lambda_sdk.testing import build_job_context, build_jobs
|
|
9
|
+
|
|
10
|
+
rc = build_job_context(config={"threshold": 5})
|
|
11
|
+
results = handle(build_jobs([{"x": 1}, {"x": 2}]), rc.context)
|
|
12
|
+
|
|
13
|
+
assert results == [...]
|
|
14
|
+
assert rc.heartbeats == 2
|
|
15
|
+
assert rc.streams["stats"] == [...]
|
|
16
|
+
assert rc.logs["logs"] == [{"severity": "info", "content": "..."}]
|
|
17
|
+
|
|
18
|
+
Works for batch handlers and for `@per_job` handlers alike -- a decorated
|
|
19
|
+
per-job handler already has the batch signature. To test the per-job function in
|
|
20
|
+
isolation, call it with a single `build_jobs([...])[0]` and `rc.context`.
|
|
21
|
+
|
|
22
|
+
For an end-to-end check through the *real* runtime, use `run_local`:
|
|
23
|
+
|
|
24
|
+
from onedata_lambda_sdk.testing import build_request, run_local
|
|
25
|
+
|
|
26
|
+
result = run_local(handle, build_request([{"x": 1}], config={"k": "v"}), out_dir=tmp_path)
|
|
27
|
+
assert result.envelope == {"resultsBatch": [...]}
|
|
28
|
+
assert result.streams["stats"] == [...]
|
|
29
|
+
|
|
30
|
+
It drives `run(handler, raw)` in dev mode and returns the response envelope plus the items
|
|
31
|
+
each `/out/<stream>` received -- exercising the seam the unit helpers skip (wire parsing,
|
|
32
|
+
`JobContext` construction, the buffered flusher writing files, envelope assembly).
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
__author__ = "Bartosz Walkowicz"
|
|
36
|
+
__copyright__ = "Copyright (C) 2026 Onedata (onedata.org)"
|
|
37
|
+
__license__ = "This software is released under the MIT license cited in LICENSE.txt"
|
|
38
|
+
|
|
39
|
+
import contextlib
|
|
40
|
+
import json
|
|
41
|
+
import os
|
|
42
|
+
from collections.abc import Callable, Iterable, Iterator
|
|
43
|
+
from dataclasses import dataclass
|
|
44
|
+
from typing import Any
|
|
45
|
+
|
|
46
|
+
from ._wire import AtmJobBatchRequestCtx
|
|
47
|
+
from .job import Job, JobContext
|
|
48
|
+
from .logging import Logger
|
|
49
|
+
from .runtime import ENV_DEV_MODE, run
|
|
50
|
+
from .streaming import ResultStreamer
|
|
51
|
+
from .types import AtmObject, LogLevel
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
# --- Unit testing: the handler in isolation ------------------------------------------
|
|
55
|
+
# Drive a handler directly against a synthetic JobContext whose services (heartbeat,
|
|
56
|
+
# streamers, loggers) record into memory instead of touching the network or `/out`.
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class _Counter:
|
|
60
|
+
"""A mutable integer cell shared between the heartbeat closure and `RecordedContext`."""
|
|
61
|
+
|
|
62
|
+
def __init__(self) -> None:
|
|
63
|
+
self.value = 0
|
|
64
|
+
|
|
65
|
+
def increment(self) -> None:
|
|
66
|
+
self.value += 1
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class _RecordingStreamer(ResultStreamer[Any]):
|
|
70
|
+
"""A streamer that captures items in a list instead of writing to `/out`."""
|
|
71
|
+
|
|
72
|
+
# Deliberately bypasses ResultStreamer.__init__ (no flusher/heartbeat/out_dir needed):
|
|
73
|
+
# both public methods are overridden to append to the sink, so no inherited code runs.
|
|
74
|
+
def __init__(self, result_name: str, sink: list[Any]) -> None:
|
|
75
|
+
self.result_name = result_name
|
|
76
|
+
self._sink = sink
|
|
77
|
+
|
|
78
|
+
def stream_item(self, item: Any) -> None:
|
|
79
|
+
self._sink.append(item)
|
|
80
|
+
|
|
81
|
+
def stream_items(self, items: Iterable[Any]) -> None:
|
|
82
|
+
self._sink.extend(items)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
class _RecordingLogger(Logger):
|
|
86
|
+
"""A logger that captures all entries (no level filtering) instead of writing out."""
|
|
87
|
+
|
|
88
|
+
# Like _RecordingStreamer, bypasses the parent __init__; stream_items is overridden to
|
|
89
|
+
# capture every entry, so the level threshold the real Logger applies is intentionally
|
|
90
|
+
# skipped -- a test sees all emitted logs regardless of `log_level`.
|
|
91
|
+
def __init__(self, result_name: str, sink: list[AtmObject]) -> None:
|
|
92
|
+
self.result_name = result_name
|
|
93
|
+
self._sink = sink
|
|
94
|
+
|
|
95
|
+
def stream_items(self, items: Iterable[AtmObject]) -> None:
|
|
96
|
+
self._sink.extend(items)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class RecordedContext:
|
|
100
|
+
"""A synthetic `JobContext` plus the side effects captured during a test."""
|
|
101
|
+
|
|
102
|
+
def __init__(
|
|
103
|
+
self,
|
|
104
|
+
context: JobContext[Any],
|
|
105
|
+
logs: dict[str, list[AtmObject]],
|
|
106
|
+
streams: dict[str, list[Any]],
|
|
107
|
+
heartbeats: _Counter,
|
|
108
|
+
) -> None:
|
|
109
|
+
self.context = context
|
|
110
|
+
# Log entries the handler emitted, keyed by log stream name (captured regardless of
|
|
111
|
+
# level -- the recording logger does not filter; see _RecordingLogger).
|
|
112
|
+
self.logs = logs
|
|
113
|
+
# Streamed items keyed by stream name.
|
|
114
|
+
self.streams = streams
|
|
115
|
+
self._heartbeats = heartbeats
|
|
116
|
+
|
|
117
|
+
@property
|
|
118
|
+
def heartbeats(self) -> int:
|
|
119
|
+
"""
|
|
120
|
+
How many times the handler called `ctx.heartbeat()` explicitly.
|
|
121
|
+
|
|
122
|
+
Unlike the real runtime, the recording streamers/loggers do not emit heartbeats,
|
|
123
|
+
so this counts only direct `ctx.heartbeat()` calls.
|
|
124
|
+
"""
|
|
125
|
+
return self._heartbeats.value
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def build_jobs(args_batch: list[Any]) -> list[Job[Any]]:
|
|
129
|
+
"""Wrap a list of argument objects into `Job`s with synthetic trace ids."""
|
|
130
|
+
return [Job(args=args, trace_id=f"test-{i}") for i, args in enumerate(args_batch)]
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def build_job_context(
|
|
134
|
+
*,
|
|
135
|
+
config: Any = None,
|
|
136
|
+
log_level: LogLevel = "info",
|
|
137
|
+
user_id: str = "test-user",
|
|
138
|
+
space_id: str = "test-space",
|
|
139
|
+
workflow_execution_id: str = "test-workflow-execution",
|
|
140
|
+
oneprovider_id: str = "test-oneprovider",
|
|
141
|
+
oneprovider_domain: str = "oneprovider.test",
|
|
142
|
+
onezone_domain: str = "onezone.test",
|
|
143
|
+
access_token: str = "test-token",
|
|
144
|
+
timeout_seconds: int = 120,
|
|
145
|
+
) -> RecordedContext:
|
|
146
|
+
"""
|
|
147
|
+
Build a synthetic `JobContext` whose services record into memory.
|
|
148
|
+
|
|
149
|
+
Returns a `RecordedContext` exposing the context (`.context`) plus the captured
|
|
150
|
+
`.logs`, `.streams` and `.heartbeats`.
|
|
151
|
+
|
|
152
|
+
Note: captured logs are *not* level-filtered -- `log_level` is stored on the context,
|
|
153
|
+
but the recording logger keeps every entry, so a test sees all emitted logs. To
|
|
154
|
+
exercise real level filtering, go through `run_local`.
|
|
155
|
+
"""
|
|
156
|
+
logs: dict[str, list[AtmObject]] = {}
|
|
157
|
+
streams: dict[str, list[Any]] = {}
|
|
158
|
+
heartbeats = _Counter()
|
|
159
|
+
|
|
160
|
+
def heartbeat() -> None:
|
|
161
|
+
heartbeats.increment()
|
|
162
|
+
|
|
163
|
+
streamers: dict[str, _RecordingStreamer] = {}
|
|
164
|
+
|
|
165
|
+
def result_streamer_factory(name: str, *, buffered: bool = True) -> ResultStreamer[Any]:
|
|
166
|
+
streamer = streamers.get(name)
|
|
167
|
+
if streamer is None:
|
|
168
|
+
streamer = _RecordingStreamer(name, streams.setdefault(name, []))
|
|
169
|
+
streamers[name] = streamer
|
|
170
|
+
return streamer
|
|
171
|
+
|
|
172
|
+
loggers: dict[str, _RecordingLogger] = {}
|
|
173
|
+
|
|
174
|
+
def logger_factory(name: str) -> Logger:
|
|
175
|
+
logger = loggers.get(name)
|
|
176
|
+
if logger is None:
|
|
177
|
+
logger = _RecordingLogger(name, logs.setdefault(name, []))
|
|
178
|
+
loggers[name] = logger
|
|
179
|
+
return logger
|
|
180
|
+
|
|
181
|
+
raw: AtmJobBatchRequestCtx[Any] = AtmJobBatchRequestCtx(
|
|
182
|
+
userId=user_id,
|
|
183
|
+
spaceId=space_id,
|
|
184
|
+
atmWorkflowExecutionId=workflow_execution_id,
|
|
185
|
+
oneproviderId=oneprovider_id,
|
|
186
|
+
oneproviderDomain=oneprovider_domain,
|
|
187
|
+
onezoneDomain=onezone_domain,
|
|
188
|
+
accessToken=access_token,
|
|
189
|
+
heartbeatUrl="",
|
|
190
|
+
timeoutSeconds=timeout_seconds,
|
|
191
|
+
logLevel=log_level,
|
|
192
|
+
config=config,
|
|
193
|
+
)
|
|
194
|
+
|
|
195
|
+
context: JobContext[Any] = JobContext(
|
|
196
|
+
config=config,
|
|
197
|
+
user_id=user_id,
|
|
198
|
+
space_id=space_id,
|
|
199
|
+
workflow_execution_id=workflow_execution_id,
|
|
200
|
+
oneprovider_id=oneprovider_id,
|
|
201
|
+
oneprovider_domain=oneprovider_domain,
|
|
202
|
+
onezone_domain=onezone_domain,
|
|
203
|
+
access_token=access_token,
|
|
204
|
+
timeout_seconds=timeout_seconds,
|
|
205
|
+
log_level=log_level,
|
|
206
|
+
heartbeat=heartbeat,
|
|
207
|
+
result_streamer_factory=result_streamer_factory,
|
|
208
|
+
logger_factory=logger_factory,
|
|
209
|
+
raw=raw,
|
|
210
|
+
)
|
|
211
|
+
return RecordedContext(context, logs, streams, heartbeats)
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
# --- Integration testing: through the real run() -------------------------------------
|
|
215
|
+
# Drive the real runtime end-to-end in dev mode against an in-memory wire request and
|
|
216
|
+
# capture what reached the response envelope and the `/out/<stream>` files.
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
@dataclass
|
|
220
|
+
class LocalRunResult:
|
|
221
|
+
"""The outcome of `run_local`: the response envelope plus captured stream contents."""
|
|
222
|
+
|
|
223
|
+
# The dict `run()` returned -- `{"resultsBatch": [...]}` or `{"exception": ...}`.
|
|
224
|
+
envelope: dict[str, Any]
|
|
225
|
+
# Items written to each `/out/<name>` stream during the run (parsed JSON lines), by name.
|
|
226
|
+
streams: dict[str, list[Any]]
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def build_request(
|
|
230
|
+
args_batch: list[Any],
|
|
231
|
+
*,
|
|
232
|
+
config: Any = None,
|
|
233
|
+
log_level: LogLevel = "info",
|
|
234
|
+
user_id: str = "test-user",
|
|
235
|
+
space_id: str = "test-space",
|
|
236
|
+
workflow_execution_id: str = "test-workflow-execution",
|
|
237
|
+
oneprovider_id: str = "test-oneprovider",
|
|
238
|
+
oneprovider_domain: str = "oneprovider.test",
|
|
239
|
+
onezone_domain: str = "onezone.test",
|
|
240
|
+
access_token: str = "test-token",
|
|
241
|
+
heartbeat_url: str = "https://heartbeat.test",
|
|
242
|
+
timeout_seconds: int = 120,
|
|
243
|
+
) -> dict[str, Any]:
|
|
244
|
+
"""
|
|
245
|
+
Build a raw wire request (the dict `run()` parses), with a per-job `__meta.traceId`.
|
|
246
|
+
|
|
247
|
+
The integration-layer counterpart to `build_jobs`/`build_job_context`: pass the result to
|
|
248
|
+
`run_local` (or `json.dumps` it and call `run` yourself). Each item of `args_batch` is one
|
|
249
|
+
job's arguments; the `__meta` envelope the framework adds on the wire is injected here.
|
|
250
|
+
"""
|
|
251
|
+
return {
|
|
252
|
+
"ctx": {
|
|
253
|
+
"userId": user_id,
|
|
254
|
+
"spaceId": space_id,
|
|
255
|
+
"atmWorkflowExecutionId": workflow_execution_id,
|
|
256
|
+
"oneproviderId": oneprovider_id,
|
|
257
|
+
"oneproviderDomain": oneprovider_domain,
|
|
258
|
+
"onezoneDomain": onezone_domain,
|
|
259
|
+
"accessToken": access_token,
|
|
260
|
+
"heartbeatUrl": heartbeat_url,
|
|
261
|
+
"timeoutSeconds": timeout_seconds,
|
|
262
|
+
"logLevel": log_level,
|
|
263
|
+
"config": config,
|
|
264
|
+
},
|
|
265
|
+
"argsBatch": [
|
|
266
|
+
{**args, "__meta": {"traceId": f"test-{i}"}} for i, args in enumerate(args_batch)
|
|
267
|
+
],
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def run_local(
|
|
272
|
+
handler: Callable[..., Any],
|
|
273
|
+
request: dict[str, Any] | str,
|
|
274
|
+
*,
|
|
275
|
+
out_dir: str | os.PathLike[str],
|
|
276
|
+
) -> LocalRunResult:
|
|
277
|
+
"""
|
|
278
|
+
Drive the real runtime `run(handler, raw)` in dev mode and capture its side effects.
|
|
279
|
+
|
|
280
|
+
Runs in dev mode (`ONEDATA_LAMBDA_DEV=1`: heartbeats logged not POSTed, no stdout
|
|
281
|
+
redirect, no Oneclient mount-wait) with the streaming output directory pointed at
|
|
282
|
+
`out_dir` (pass a pytest `tmp_path`). Returns the response envelope plus the items each
|
|
283
|
+
stream received, so an integration test can assert on both -- the seam unit tests skip:
|
|
284
|
+
wire parsing, `JobContext` construction, the buffered flusher writing files, and
|
|
285
|
+
envelope assembly.
|
|
286
|
+
|
|
287
|
+
`request` is a wire request dict (see `build_request`) or a raw JSON string.
|
|
288
|
+
"""
|
|
289
|
+
raw = request if isinstance(request, str) else json.dumps(request)
|
|
290
|
+
with _dev_mode():
|
|
291
|
+
envelope = run(handler, raw, out_dir=os.fspath(out_dir))
|
|
292
|
+
return LocalRunResult(envelope=dict(envelope), streams=_read_streams(out_dir))
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
@contextlib.contextmanager
|
|
296
|
+
def _dev_mode() -> Iterator[None]:
|
|
297
|
+
previous = os.environ.get(ENV_DEV_MODE)
|
|
298
|
+
os.environ[ENV_DEV_MODE] = "1"
|
|
299
|
+
try:
|
|
300
|
+
yield
|
|
301
|
+
finally:
|
|
302
|
+
if previous is None:
|
|
303
|
+
os.environ.pop(ENV_DEV_MODE, None)
|
|
304
|
+
else:
|
|
305
|
+
os.environ[ENV_DEV_MODE] = previous
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def _read_streams(out_dir: str | os.PathLike[str]) -> dict[str, list[Any]]:
|
|
309
|
+
streams: dict[str, list[Any]] = {}
|
|
310
|
+
for name in sorted(os.listdir(out_dir)):
|
|
311
|
+
path = os.path.join(os.fspath(out_dir), name)
|
|
312
|
+
if not os.path.isfile(path):
|
|
313
|
+
continue
|
|
314
|
+
with open(path) as stream_file:
|
|
315
|
+
streams[name] = [json.loads(line) for line in stream_file if line.strip()]
|
|
316
|
+
return streams
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Public data types for Onedata Automation lambdas: the JSON shapes of Onedata entities a
|
|
3
|
+
handler receives or produces (`AtmFile`, `AtmDataset`, `AtmObject`, …), mirroring the
|
|
4
|
+
public Onedata API, plus `AtmException` (the per-job failure marker).
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
__author__ = "Bartosz Walkowicz"
|
|
8
|
+
__copyright__ = "Copyright (C) 2022-2026 Onedata (onedata.org)"
|
|
9
|
+
__license__ = "This software is released under the MIT license cited in LICENSE.txt"
|
|
10
|
+
|
|
11
|
+
from typing import Any, Literal, NotRequired, TypedDict
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
##===================================================================
|
|
15
|
+
## Basic automation data types
|
|
16
|
+
##===================================================================
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
type FileType = Literal["REG", "DIR", "SYMLNK"]
|
|
20
|
+
type ProtectionFlag = Literal["data_protection", "metadata_protection"]
|
|
21
|
+
type InheritancePath = Literal["none", "direct", "ancestor", "direct_and_ancestor"]
|
|
22
|
+
type LogLevel = Literal[
|
|
23
|
+
"debug",
|
|
24
|
+
"info",
|
|
25
|
+
"notice",
|
|
26
|
+
"warning",
|
|
27
|
+
"error",
|
|
28
|
+
"critical",
|
|
29
|
+
"alert",
|
|
30
|
+
"emergency",
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
type AtmObject = dict[str, Any]
|
|
34
|
+
"""Generic JSON object."""
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class AtmDataset(TypedDict):
|
|
38
|
+
"""
|
|
39
|
+
JSON object describing dataset.
|
|
40
|
+
|
|
41
|
+
Refer to the API specification for more information:
|
|
42
|
+
https://onedata.org/#/home/api/latest/oneprovider?anchor=operation/get_dataset
|
|
43
|
+
"""
|
|
44
|
+
|
|
45
|
+
state: Literal["attached", "detached"]
|
|
46
|
+
datasetId: str
|
|
47
|
+
parentId: str | None
|
|
48
|
+
rootFileId: str
|
|
49
|
+
rootFileType: FileType
|
|
50
|
+
rootFilePath: str
|
|
51
|
+
rootFileDeleted: bool
|
|
52
|
+
protectionFlags: list[ProtectionFlag]
|
|
53
|
+
effectiveProtectionFlags: list[ProtectionFlag]
|
|
54
|
+
creationTime: int
|
|
55
|
+
archiveCount: int
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class AtmFile(TypedDict, total=False):
|
|
59
|
+
"""
|
|
60
|
+
JSON object describing a file in Onedata filesystem.
|
|
61
|
+
|
|
62
|
+
Refer to the API specification for more information:
|
|
63
|
+
https://onedata.org/#/home/api/latest/oneprovider?anchor=operation/get_attrs
|
|
64
|
+
|
|
65
|
+
NOTE: dynamic fields like `xattr.*` are not typed due to limitations
|
|
66
|
+
of typing machinery.
|
|
67
|
+
"""
|
|
68
|
+
|
|
69
|
+
fileId: str
|
|
70
|
+
index: str
|
|
71
|
+
type: FileType
|
|
72
|
+
activePermissionsType: Literal["posix", "acl"]
|
|
73
|
+
posixPermissions: str
|
|
74
|
+
acl: list[dict]
|
|
75
|
+
name: str
|
|
76
|
+
conflictingName: str | None
|
|
77
|
+
path: str
|
|
78
|
+
parentFileId: str | None
|
|
79
|
+
displayGid: int
|
|
80
|
+
displayUid: int
|
|
81
|
+
atime: int
|
|
82
|
+
mtime: int
|
|
83
|
+
ctime: int
|
|
84
|
+
size: int | None
|
|
85
|
+
isFullyReplicatedLocally: bool | None
|
|
86
|
+
localReplicationRate: float
|
|
87
|
+
originProviderId: str
|
|
88
|
+
directShareIds: list[str]
|
|
89
|
+
ownerUserId: str
|
|
90
|
+
hardlinkCount: int
|
|
91
|
+
symlinkValue: str | None
|
|
92
|
+
hasCustomMetadata: bool
|
|
93
|
+
effProtectionFlags: list[ProtectionFlag]
|
|
94
|
+
effDatasetProtectionFlags: list[ProtectionFlag]
|
|
95
|
+
effDatasetInheritancePath: InheritancePath
|
|
96
|
+
effQosInheritancePath: InheritancePath
|
|
97
|
+
aggregateQosStatus: Literal["fulfilled", "pending", "impossible"]
|
|
98
|
+
archiveRecallRootFileId: str | None
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
class AtmGroup(TypedDict):
|
|
102
|
+
"""
|
|
103
|
+
JSON object describing group.
|
|
104
|
+
|
|
105
|
+
Refer to the API specification for more information:
|
|
106
|
+
https://onedata.org/#/home/api/latest/onezone?anchor=operation/get_group
|
|
107
|
+
"""
|
|
108
|
+
|
|
109
|
+
groupId: str
|
|
110
|
+
name: str
|
|
111
|
+
type: Literal["organization", "unit", "team", "role_holders"]
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
class AtmRange(TypedDict):
|
|
115
|
+
"""
|
|
116
|
+
JSON object describing a sequence of integers.
|
|
117
|
+
|
|
118
|
+
AtmRange is a sequence of integers produced from the start to end by step.
|
|
119
|
+
|
|
120
|
+
Fields
|
|
121
|
+
--------
|
|
122
|
+
end
|
|
123
|
+
Value for which the sequence of integers will exclusively end.
|
|
124
|
+
If step value is positive, end value must be higher than start value.
|
|
125
|
+
Otherwise end value must be lower than start value.
|
|
126
|
+
start
|
|
127
|
+
Starting value for which the sequence of integers will inclusively start.
|
|
128
|
+
If step value is positive, start value must be lower than end value.
|
|
129
|
+
Otherwise start value must be higher than end value.
|
|
130
|
+
By default, it's set to: 0.
|
|
131
|
+
step
|
|
132
|
+
Value which specifies the size of incrementation or decrementation.
|
|
133
|
+
It cannot be equal 0, and by default, it's set to: 1.
|
|
134
|
+
|
|
135
|
+
Examples
|
|
136
|
+
--------
|
|
137
|
+
AtmRange(start=10, end=20, step=2) -> [10, 12, 14, 16, 18]
|
|
138
|
+
AtmRange(start=10, end=-10, step=-4) -> [10, 6, 2, -2, -6]
|
|
139
|
+
AtmRange(end=10, step=6) -> [0, 6]
|
|
140
|
+
AtmRange(end=5) -> [0, 1, 2, 3, 4]
|
|
141
|
+
AtmRange(start=4, end=-1, step=-1) -> [4, 3, 2, 1, 0]
|
|
142
|
+
AtmRange(start=5, end=5, step=-2) -> []
|
|
143
|
+
AtmRange(start=4, end=5, step=-2) -> Invalid!
|
|
144
|
+
"""
|
|
145
|
+
|
|
146
|
+
end: int
|
|
147
|
+
start: NotRequired[int]
|
|
148
|
+
step: NotRequired[int]
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
class AtmTimeSeriesMeasurement(TypedDict):
|
|
152
|
+
"""
|
|
153
|
+
JSON object describing time series measurement.
|
|
154
|
+
|
|
155
|
+
Time series measurement is a named value-timestamp pair which expresses some
|
|
156
|
+
property that can be dynamically changing (like file size, network speed, etc.).
|
|
157
|
+
Such measurements can be aggregated within time series stores and
|
|
158
|
+
serve as a data source for GUI charts.
|
|
159
|
+
"""
|
|
160
|
+
|
|
161
|
+
tsName: str
|
|
162
|
+
timestamp: int
|
|
163
|
+
value: float
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
class AtmException(TypedDict):
|
|
167
|
+
"""
|
|
168
|
+
The exception envelope -- a failure reported on the wire as `{"exception": ...}`.
|
|
169
|
+
|
|
170
|
+
One shape, used at two levels:
|
|
171
|
+
|
|
172
|
+
* **per-job** -- returned in a handler's results list (in place of a result) to fail a
|
|
173
|
+
single job without failing the whole batch; `run()` passes it through.
|
|
174
|
+
* **whole-batch** -- returned by `run()` as the entire response (replacing
|
|
175
|
+
`resultsBatch`) when the batch fails before producing per-job results -- a malformed
|
|
176
|
+
request, a failed Oneclient mount, or a batch-level handler exception.
|
|
177
|
+
"""
|
|
178
|
+
|
|
179
|
+
exception: float | str | AtmObject
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: onedata-lambda-sdk
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: SDK for writing Onedata Automation lambdas
|
|
5
|
+
Project-URL: Homepage, https://onedata.org
|
|
6
|
+
Author: Bartosz Walkowicz
|
|
7
|
+
License: MIT License
|
|
8
|
+
===========
|
|
9
|
+
|
|
10
|
+
Copyright (C) 2022: Onedata (onedata.org)
|
|
11
|
+
|
|
12
|
+
Permission is hereby granted, free of charge, to any person
|
|
13
|
+
obtaining a copy of this software and associated documentation
|
|
14
|
+
files (the "Software"), to deal in the Software without
|
|
15
|
+
restriction, including without limitation the rights to use, copy,
|
|
16
|
+
modify, merge, publish, distribute, sublicense, and/or sell copies
|
|
17
|
+
of the Software, and to permit persons to whom the Software is
|
|
18
|
+
furnished to do so, subject to the following conditions:
|
|
19
|
+
|
|
20
|
+
The above copyright notice and this permission notice shall be
|
|
21
|
+
included in all copies or substantial portions of the Software.
|
|
22
|
+
|
|
23
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
|
24
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
|
25
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
|
26
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
|
27
|
+
HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
|
28
|
+
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
29
|
+
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
|
30
|
+
OTHER DEALINGS IN THE SOFTWARE.
|
|
31
|
+
License-File: LICENSE.txt
|
|
32
|
+
Keywords: automation,lambda,onedata,openfaas
|
|
33
|
+
Classifier: Intended Audience :: Developers
|
|
34
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
35
|
+
Classifier: Operating System :: OS Independent
|
|
36
|
+
Classifier: Programming Language :: Python :: 3
|
|
37
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
38
|
+
Classifier: Typing :: Typed
|
|
39
|
+
Requires-Python: >=3.12
|
|
40
|
+
Requires-Dist: requests>=2.32
|
|
41
|
+
Description-Content-Type: text/markdown
|
|
42
|
+
|
|
43
|
+
# onedata-lambda-sdk
|
|
44
|
+
|
|
45
|
+
The Python SDK for writing **Onedata Automation lambdas** — the containerized
|
|
46
|
+
operations that run as job executors in automation workflows. You write one
|
|
47
|
+
handler function; the SDK turns the raw OpenFaaS request into typed jobs, runs
|
|
48
|
+
your handler, streams its results, sends the mandatory first heartbeat and keeps
|
|
49
|
+
the batch alive while it streams, and shapes the response the provider expects.
|
|
50
|
+
|
|
51
|
+
```python
|
|
52
|
+
from onedata_lambda_sdk import Job, JobContext, per_job
|
|
53
|
+
|
|
54
|
+
@per_job(max_workers=10)
|
|
55
|
+
def handle(job: Job[dict], ctx: JobContext[dict]) -> dict:
|
|
56
|
+
return {"result": process(job.args)}
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
## Documentation
|
|
60
|
+
|
|
61
|
+
The full authoring documentation lives in [`docs/`](docs/):
|
|
62
|
+
|
|
63
|
+
- **[SDK overview](docs/_overview.md)** — the three-layer model (base image →
|
|
64
|
+
SDK → handler) and how a job batch flows.
|
|
65
|
+
- **Guides** — [writing a handler](docs/guides/writing-a-handler.md),
|
|
66
|
+
[a single-lambda repo](docs/guides/single-lambda-repo.md),
|
|
67
|
+
[several lambdas in a uv workspace](docs/guides/shared-code-uv-workspace.md),
|
|
68
|
+
[developing against a vendored SDK wheel](docs/guides/local-sdk-vendored-wheel.md),
|
|
69
|
+
[testing](docs/guides/testing-a-lambda.md),
|
|
70
|
+
[streaming logs and stats](docs/guides/streaming-logs-and-stats.md), and
|
|
71
|
+
[file access](docs/guides/file-access.md).
|
|
72
|
+
- **[Runtime internals](docs/internals/runtime-and-lifecycle.md)** — for SDK
|
|
73
|
+
maintainers.
|
|
74
|
+
|
|
75
|
+
The platform-side contract this SDK implements — how Oneprovider invokes a
|
|
76
|
+
lambda, the I/O and relay methods, heartbeats — is in the internal developer
|
|
77
|
+
documentation under `docs/automation/internals/lambda/` in
|
|
78
|
+
`onedata-dev-documentation`.
|
|
79
|
+
|
|
80
|
+
## Installation
|
|
81
|
+
|
|
82
|
+
Requires Python 3.12 or newer.
|
|
83
|
+
|
|
84
|
+
```bash
|
|
85
|
+
pip install onedata-lambda-sdk
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
## Development
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
make sync # create the .venv from the lockfile
|
|
92
|
+
make check # lint (ruff format-check + ruff check + mypy) + tests
|
|
93
|
+
make test # pytest with coverage
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
Run `make help` for the full target list.
|
|
97
|
+
|
|
98
|
+
## Compatibility
|
|
99
|
+
|
|
100
|
+
`onedata-lambda-sdk` targets the **v3 lambda model** (`run` / `@per_job` /
|
|
101
|
+
`Job` / `JobContext`) and requires Oneprovider `21.02.5` or newer. A v3 lambda
|
|
102
|
+
builds on the `onedata/lambda-base-slim:v3` base image — see
|
|
103
|
+
[Lambda generations](docs/_overview.md#lambda-generations-v2-and-v3).
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
onedata_lambda_sdk/__init__.py,sha256=oRrig3MXT4dY8CEkPGWcuk1X9KqDGbpke7SEMV-Infc,1603
|
|
2
|
+
onedata_lambda_sdk/_wire.py,sha256=qllskLbtVbdvnDZrVn0gekxSr1HiHv6KvuoLvxSKKm8,3566
|
|
3
|
+
onedata_lambda_sdk/job.py,sha256=AimjlFTewBh5pcxe6Pt4aQHGUdcAQzuhEwxL5in9sTc,4528
|
|
4
|
+
onedata_lambda_sdk/logging.py,sha256=Ar4Knkz7930cQTrMRB8G0AWypbAulRfJ_x6UrZ1ql2Q,3057
|
|
5
|
+
onedata_lambda_sdk/oneclient.py,sha256=oK0jqM1-FOzlklF6SzZA95Uv9OZoBam588m7v01PWV8,1774
|
|
6
|
+
onedata_lambda_sdk/perjob.py,sha256=-u_sbbllPm7B5hDwwVsO-OR8B_R4tDrUBl8NFPDPAE8,4991
|
|
7
|
+
onedata_lambda_sdk/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
|
+
onedata_lambda_sdk/runtime.py,sha256=K-PsbUenOJLny37gPTgbxI6VzPnxCzTBiMegGrT5HyU,15863
|
|
9
|
+
onedata_lambda_sdk/stats.py,sha256=7gEDRInQv3Eq_PPahxNX244gAD4g5UL0VpILYb9JmsI,1567
|
|
10
|
+
onedata_lambda_sdk/streaming.py,sha256=tgeOTF6bJ4HThOb92kVcLz41k9gAwBlScGAdI_C-lQI,5461
|
|
11
|
+
onedata_lambda_sdk/testing.py,sha256=3o59sAMmZiBH-fs0GYd5rqGjaOWCJmggA71QbtmxvSM,11648
|
|
12
|
+
onedata_lambda_sdk/types.py,sha256=7YPT3-O4A2GQ5--9-amY2scWB2cKL9_i-YbKRLL4Z8Y,5508
|
|
13
|
+
onedata_lambda_sdk-1.0.0.dist-info/METADATA,sha256=AEDfBd-DXfAWVB3MAadQxLklt2BLqRmHijclmnNyOaQ,4229
|
|
14
|
+
onedata_lambda_sdk-1.0.0.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
|
|
15
|
+
onedata_lambda_sdk-1.0.0.dist-info/licenses/LICENSE.txt,sha256=sNMlunt8MCfOBHX09_CXxyc1k_O9rA7NhlDfECkp4Es,1163
|
|
16
|
+
onedata_lambda_sdk-1.0.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
===========
|
|
3
|
+
|
|
4
|
+
Copyright (C) 2022: Onedata (onedata.org)
|
|
5
|
+
|
|
6
|
+
Permission is hereby granted, free of charge, to any person
|
|
7
|
+
obtaining a copy of this software and associated documentation
|
|
8
|
+
files (the "Software"), to deal in the Software without
|
|
9
|
+
restriction, including without limitation the rights to use, copy,
|
|
10
|
+
modify, merge, publish, distribute, sublicense, and/or sell copies
|
|
11
|
+
of the Software, and to permit persons to whom the Software is
|
|
12
|
+
furnished to do so, subject to the following conditions:
|
|
13
|
+
|
|
14
|
+
The above copyright notice and this permission notice shall be
|
|
15
|
+
included in all copies or substantial portions of the Software.
|
|
16
|
+
|
|
17
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
|
18
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
|
19
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
|
20
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
|
21
|
+
HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
|
22
|
+
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
23
|
+
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
|
24
|
+
OTHER DEALINGS IN THE SOFTWARE.
|