onedata-lambda-sdk 1.0.0__py3-none-any.whl

onedata_lambda_sdk/__init__.py

@@ -0,0 +1,60 @@
+"""
+SDK for writing Onedata Automation lambdas.
+
+Public surface: the handler API (`Job`, `JobContext`, `JobException`), the contract
+runtime (`run`, `per_job`), the Onedata wire/contract field types (the `Atm*` names),
+the time-series measurement builder, and the mounted-Oneclient helpers.
+
+Naming convention: `Atm*` marks the Onedata Automation *contract* -- the JSON shapes that
+cross the wire to/from the provider (`AtmFile`, `AtmException`, …). Everything the SDK adds
+purely as authoring ergonomics (`Job`, `JobContext`, `JobException`, `ResultStreamer`,
+`TimeSeriesMeasurementBuilder`, …) is SDK-local and carries no prefix.
+"""
+
+from .job import Job, JobContext, JobException
+from .oneclient import mount_point, mounted_file_path
+from .perjob import DEFAULT_MAX_WORKERS, per_job
+from .runtime import run
+from .stats import TimeSeriesMeasurementBuilder
+from .types import (
+    AtmDataset,
+    AtmException,
+    AtmFile,
+    AtmGroup,
+    AtmObject,
+    AtmRange,
+    AtmTimeSeriesMeasurement,
+    FileType,
+    InheritancePath,
+    LogLevel,
+    ProtectionFlag,
+)
+
+
+__all__ = [
+    # runtime
+    "run",
+    "per_job",
+    "DEFAULT_MAX_WORKERS",
+    # handler API
+    "Job",
+    "JobContext",
+    "JobException",
+    # mounted Oneclient helpers
+    "mount_point",
+    "mounted_file_path",
+    # wire/contract field types (Atm*)
+    "AtmException",
+    "AtmObject",
+    "AtmDataset",
+    "AtmFile",
+    "AtmGroup",
+    "AtmRange",
+    "AtmTimeSeriesMeasurement",
+    "FileType",
+    "ProtectionFlag",
+    "InheritancePath",
+    "LogLevel",
+    # builders
+    "TimeSeriesMeasurementBuilder",
+]

onedata_lambda_sdk/_wire.py

@@ -0,0 +1,111 @@
+"""
+Internal wire types: the raw JSON shapes exchanged between Oneprovider and a lambda.
+
+These model the OpenFaaS job-batch request/response protocol verbatim (camelCase, as
+sent on the wire). They are an implementation detail parsed by `run()` and are NOT
+part of the public handler API -- handlers work with `Job` / `JobContext`
+instead (see `job`). The full request context remains reachable as an escape hatch
+via `JobContext.raw`.
+"""
+
+__author__ = "Bartosz Walkowicz"
+__copyright__ = "Copyright (C) 2026 Onedata (onedata.org)"
+__license__ = "This software is released under the MIT license cited in LICENSE.txt"
+
+from collections.abc import Sequence
+from typing import TypedDict
+
+from .types import AtmException, LogLevel
+
+
+class AtmJobBatchRequestCtx[C](TypedDict):
+    """
+    JSON object describing job batch request context.
+
+    Fields
+    --------
+    userId
+        Id of the scheduling user.
+    spaceId
+        Id of the space in context of which the workflow is executed.
+    atmWorkflowExecutionId
+        Workflow execution Id.
+    oneproviderDomain
+        Domain of job scheduling Oneprovider.
+    oneproviderId
+        Id of job scheduling Oneprovider.
+    onezoneDomain
+        The domain of the Onezone service that manages the Onedata environment.
+    accessToken
+        Token used to authorize operations in Onedata.
+
+        Refer to the API specification for more information:
+        https://onedata.org/#/home/documentation/latest/doc/using_onedata/tokens[access-tokens].html
+    heartbeatUrl
+        The target url for sending heartbeats.
+    timeoutSeconds
+        The time interval from last heartbeat after which job batch is marked as failed.
+    logLevel
+        Level controlling the amount of information recorded in audit logs as only logs
+        with severity equal or higher to this level will be stored.
+    config
+        User defined task configuration.
+    """
+
+    userId: str
+    spaceId: str
+    atmWorkflowExecutionId: str
+    oneproviderDomain: str
+    oneproviderId: str
+    onezoneDomain: str
+    accessToken: str
+    heartbeatUrl: str
+    timeoutSeconds: int
+    logLevel: LogLevel
+    config: C
+
+
+class AtmJobMeta(TypedDict):
+    """Per-job metadata the framework attaches to each job's args object on the wire."""
+
+    traceId: str
+
+
+# Functional syntax because `__meta` is a dunder key (a class-body annotation would be
+# name-mangled). Each wire args object also carries the lambda's own argument keys
+# alongside `__meta`; the runtime strips `__meta` off (-> Job.trace_id) and hands the
+# rest to the handler as the job's args.
+AtmJobArgs = TypedDict("AtmJobArgs", {"__meta": AtmJobMeta})
+
+
+class AtmJobBatchRequest[C](TypedDict):
+    """
+    JSON object describing job batch request.
+
+    Fields
+    --------
+    ctx
+        Context of job batch request.
+    argsBatch
+        List of per-job args objects. Each carries a `__meta` envelope (the job's
+        `traceId`) alongside the lambda's own input arguments.
+    """
+
+    ctx: AtmJobBatchRequestCtx[C]
+    argsBatch: list[AtmJobArgs]
+
+
+class AtmJobBatchResponse[R](TypedDict):
+    """
+    JSON object describing job batch response.
+
+    Fields
+    --------
+    resultsBatch
+        List of JSON objects, each carrying results of a single job (lambda output).
+        The length of this list must be equal to the length of the corresponding
+        AtmJobBatchRequest.argsBatch. However, if the lambda returns empty results for
+        each job, the resultsBatch can be set to None.
+    """
+
+    resultsBatch: Sequence[AtmException | R | None] | None

onedata_lambda_sdk/job.py

@@ -0,0 +1,119 @@
+"""
+The handler API: the per-job unit (`Job`), the shared batch context (`JobContext`), and
+the failure signal (`JobException`) -- what a handler signs against.
+
+These are SDK-local authoring abstractions, not the wire contract, so they carry no `Atm`
+prefix (unlike `AtmFile`, `AtmException`, … which mirror what crosses the wire).
+"""
+
+__author__ = "Bartosz Walkowicz"
+__copyright__ = "Copyright (C) 2026 Onedata (onedata.org)"
+__license__ = "This software is released under the MIT license cited in LICENSE.txt"
+
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from .types import LogLevel
+
+
+if TYPE_CHECKING:
+    from ._wire import AtmJobBatchRequestCtx
+    from .logging import Logger
+    from .streaming import ResultStreamer
+
+
+class JobException(Exception):
+    """
+    Raised by a handler to fail a job with a known, message-only error.
+
+    The SDK turns it into an `AtmException` entry carrying just `str(exception)` -- no
+    traceback -- so an expected failure (bad config, a failed REST call)
+    reads cleanly in the results. Any *other* exception is treated as an unexpected bug
+    and reported with a full traceback instead.
+    """
+
+
+@dataclass(frozen=True)
+class Job[A]:
+    """
+    A single unit of work passed to a handler.
+
+    `args` is the job's input (the lambda's typed arguments). `trace_id` identifies this
+    job across logs and streamed results; it comes from the wire (the framework assigns
+    it) and lives on the job -- not on the shared context, which is common to all jobs of
+    a batch, including in parallel.
+    """
+
+    args: A
+    trace_id: str
+
+
+class JobContext[C]:
+    """
+    The shared, per-batch context handed to a handler alongside its jobs.
+
+    Carries the (typed) task `config`, read-only identity/infrastructure fields, and the
+    runtime services. Services (`logger(name)`, `heartbeat()`, `result_streamer(name)`)
+    are vended by the context rather than imported globally, so they can hook into the
+    runtime (heartbeat-on-flush, the shared streaming flusher, dev-mode). They are
+    thread-safe; per-job state must never be stored here -- it travels on `Job`.
+
+    There is no default log stream: a lambda picks its stream names itself (and declares
+    them in its workflow schema), so `logger(name)` requires the name explicitly -- just
+    like `result_streamer(name)`. "logs" is only a common convention, not a requirement.
+
+    `raw` is an escape hatch to the full wire request context for fields not surfaced
+    here. The runtime (`run()`) constructs this; handlers only read it.
+    """
+
+    def __init__(
+        self,
+        *,
+        config: C,
+        user_id: str,
+        space_id: str,
+        workflow_execution_id: str,
+        oneprovider_id: str,
+        oneprovider_domain: str,
+        onezone_domain: str,
+        access_token: str,
+        timeout_seconds: int,
+        log_level: LogLevel,
+        heartbeat: Callable[[], None],
+        result_streamer_factory: "Callable[..., ResultStreamer]",
+        logger_factory: "Callable[[str], Logger]",
+        raw: "AtmJobBatchRequestCtx[C]",
+    ) -> None:
+        self.config = config
+        self.user_id = user_id
+        self.space_id = space_id
+        self.workflow_execution_id = workflow_execution_id
+        self.oneprovider_id = oneprovider_id
+        self.oneprovider_domain = oneprovider_domain
+        self.onezone_domain = onezone_domain
+        self.access_token = access_token
+        self.timeout_seconds = timeout_seconds
+        self.log_level = log_level
+        self.raw = raw
+        self._heartbeat = heartbeat
+        self._result_streamer_factory = result_streamer_factory
+        self._logger_factory = logger_factory
+
+    def heartbeat(self) -> None:
+        """Signal progress for the current job (throttled by the runtime)."""
+        self._heartbeat()
+
+    def result_streamer(self, name: str, *, buffered: bool = True) -> "ResultStreamer":
+        """Open a result/measurement stream by name (buffered by default)."""
+        return self._result_streamer_factory(name, buffered=buffered)
+
+    def logger(self, name: str) -> "Logger":
+        """
+        Open an audit-log stream by name, filtered to the batch's log level.
+
+        The name is the lambda's choice (declare it in the workflow schema); "logs" is a
+        common convention, not a requirement. Logs are written unbuffered, so they are
+        kept even if the handler crashes mid-batch.
+        """
+        return self._logger_factory(name)

onedata_lambda_sdk/logging.py

@@ -0,0 +1,99 @@
+"""
+Audit logging for lambdas: `Logger`.
+
+A thin layer over a result stream that tags each entry with a severity and drops entries
+below the configured level. Vended by `JobContext.logger(name)`; the runtime builds it
+on the lambda's chosen log stream with the batch's `log_level` and wires it to the shared
+streaming machinery (so it inherits the streamer's thread-safety).
+"""
+
+__author__ = "Bartosz Walkowicz"
+__copyright__ = "Copyright (C) 2023-2026 Onedata (onedata.org)"
+__license__ = "This software is released under the MIT license cited in LICENSE.txt"
+
+from collections.abc import Callable, Iterable
+from typing import Any, Final
+
+from .streaming import ResultStreamer, StreamFlusher
+from .types import AtmObject
+
+
+# Severity -> numeric level. Higher number == more verbose; a logger keeps entries
+# whose level is at or below its threshold (i.e. of equal-or-higher importance).
+LOG_LEVELS: Final[dict[str, int]] = {
+    "debug": 7,
+    "info": 6,
+    "notice": 5,
+    "warning": 4,
+    "error": 3,
+    "critical": 2,
+    "alert": 1,
+    "emergency": 0,
+}
+
+DEFAULT_LOG_LEVEL: Final[str] = "info"
+
+
+def normalize_severity(severity: str) -> str:
+    return severity if severity in LOG_LEVELS else DEFAULT_LOG_LEVEL
+
+
+class Logger(ResultStreamer[AtmObject]):
+    """A severity-filtered result streamer for `{severity, content}` log entries."""
+
+    def __init__(
+        self,
+        *,
+        result_name: str,
+        buffered: bool,
+        flusher: StreamFlusher,
+        heartbeat: Callable[[], None],
+        out_dir: str | None = None,
+        log_level: str = DEFAULT_LOG_LEVEL,
+    ) -> None:
+        super().__init__(
+            result_name=result_name,
+            buffered=buffered,
+            flusher=flusher,
+            heartbeat=heartbeat,
+            out_dir=out_dir,
+        )
+        self._threshold = LOG_LEVELS[normalize_severity(log_level)]
+
+    def stream_items(self, items: Iterable[AtmObject]) -> None:
+        kept = [item for item in items if LOG_LEVELS[self._severity_of(item)] <= self._threshold]
+        super().stream_items(kept)
+
+    @staticmethod
+    def _severity_of(item: AtmObject) -> str:
+        try:
+            return normalize_severity(item["severity"])
+        except (KeyError, TypeError):
+            return DEFAULT_LOG_LEVEL
+
+    def log(self, severity: str, content: Any) -> None:
+        self.stream_item({"severity": severity, "content": content})
+
+    def debug(self, content: Any) -> None:
+        self.log("debug", content)
+
+    def info(self, content: Any) -> None:
+        self.log("info", content)
+
+    def notice(self, content: Any) -> None:
+        self.log("notice", content)
+
+    def warning(self, content: Any) -> None:
+        self.log("warning", content)
+
+    def error(self, content: Any) -> None:
+        self.log("error", content)
+
+    def critical(self, content: Any) -> None:
+        self.log("critical", content)
+
+    def alert(self, content: Any) -> None:
+        self.log("alert", content)
+
+    def emergency(self, content: Any) -> None:
+        self.log("emergency", content)

onedata_lambda_sdk/oneclient.py

@@ -0,0 +1,45 @@
+"""
+Helpers for lambdas that read files through a mounted Oneclient.
+
+The mount point is a deployment fact: the platform mounts Oneclient at
+`ONECLIENT_MOUNT_POINT` -- the same variable the runtime waits on before handing a batch
+to a mounted lambda (see `runtime._await_oneclient_mount`). Reading it here, in one place,
+together with Oneclient's id-based path convention, keeps every mounted lambda agreeing
+with the runtime instead of hardcoding a path that can silently drift.
+"""
+
+__author__ = "Bartosz Walkowicz"
+__copyright__ = "Copyright (C) 2026 Onedata (onedata.org)"
+__license__ = "This software is released under the MIT license cited in LICENSE.txt"
+
+import os
+from typing import Final
+
+
+# Environment variable carrying the Oneclient mount point (set by the platform).
+ENV_MOUNT_POINT: Final[str] = "ONECLIENT_MOUNT_POINT"
+
+# Fallback used only for local dev / tests; in production the platform always sets the env.
+DEFAULT_MOUNT_POINT: Final[str] = "/mnt/onedata"
+
+
+def mount_point() -> str:
+    """
+    The Oneclient mount point, read from the environment at call time.
+
+    Falls back to `DEFAULT_MOUNT_POINT` for local dev/tests; in production the platform
+    always sets `ONECLIENT_MOUNT_POINT`. Read at call time (not import time) so tests can
+    point the mount elsewhere via the environment.
+    """
+    return os.environ.get(ENV_MOUNT_POINT) or DEFAULT_MOUNT_POINT
+
+
+def mounted_file_path(file_id: str) -> str:
+    """
+    Absolute path of a file, by its id, under the Oneclient mount.
+
+    Oneclient exposes every file at a stable id-based path
+    (`<mount>/.__onedata__file_id__<id>`), so a lambda can open a file knowing only its id,
+    without resolving its logical location.
+    """
+    return f"{mount_point()}/.__onedata__file_id__{file_id}"

onedata_lambda_sdk/perjob.py

@@ -0,0 +1,122 @@
+"""
+The per-job handler adapter: `@per_job`.
+
+Lets a handler be written in the natural per-job style -- `handle(job, ctx) -> result`
+-- and adapts it into the batch-level handler `run()` calls, so the runtime stays
+agnostic about the style used. Sibling of `runtime.run` (the batch entry point).
+"""
+
+__author__ = "Bartosz Walkowicz"
+__copyright__ = "Copyright (C) 2022-2026 Onedata (onedata.org)"
+__license__ = "This software is released under the MIT license cited in LICENSE.txt"
+
+import functools
+import traceback
+from collections.abc import Callable
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from typing import Any
+
+from .job import Job, JobContext, JobException
+from .types import AtmException
+
+
+# A per-job handler: one job in, one result out.
+PerJobHandler = Callable[[Job[Any], JobContext[Any]], Any]
+# A batch-level handler (what the runtime calls).
+BatchHandler = Callable[[list[Job[Any]], JobContext[Any]], list[Any]]
+# A once-per-batch precondition: validate config / set up shared state before any job runs.
+Precondition = Callable[[JobContext[Any]], None]
+
+# Default worker count for parallel `@per_job`. A deliberate fixed value, *not* derived
+# from `os.cpu_count()`: that returns the host's cores and ignores the pod's cgroup CPU
+# limit (and there is no `os.process_cpu_count()` on 3.12), and peak memory grows with the
+# worker count (each job may hold a read buffer). Lambdas whose profile differs pass their
+# own `max_workers`.
+DEFAULT_MAX_WORKERS = 10
+
+
+def per_job(
+    handle: PerJobHandler | None = None,
+    *,
+    max_workers: int = 1,
+    precondition: Precondition | None = None,
+) -> Any:
+    """
+    Adapt a per-job `handle(job, ctx) -> result` into the batch handler `run()` calls.
+
+    The SDK owns the batch loop: it runs each job, turns a per-job exception into an
+    `AtmException` entry (so one bad job fails only itself), assembles results in input
+    order, and heartbeats as each job completes. Sequential by default; `max_workers > 1`
+    runs jobs on a thread pool (the context services are thread-safe) -- pass
+    `DEFAULT_MAX_WORKERS` for the SDK's standard degree of parallelism. Usable bare
+    (`@per_job`) or parameterized (`@per_job(max_workers=4)`).
+
+    `precondition`, if given, runs **once** before any job is dispatched -- the place for
+    task-level validation (e.g. a config check) or one-time setup. Raising `JobException`
+    there fails the whole batch with a single clean top-level `{"exception": ...}` (via
+    `run()`), rather than repeating a per-job failure across every job.
+    """
+
+    def decorate(handle_one: PerJobHandler) -> BatchHandler:
+        # Copy name/qualname/doc/module so tracebacks and introspection point at the
+        # user's handler -- but NOT __annotations__: run_batch has the batch signature
+        # (jobs, ctx), which differs from handle_one's per-job (job, ctx).
+        @functools.wraps(
+            handle_one,
+            assigned=("__module__", "__name__", "__qualname__", "__doc__"),
+        )
+        def run_batch(jobs: list[Job[Any]], ctx: JobContext[Any]) -> list[Any]:
+            if precondition is not None:
+                # Runs once for the batch; a JobException here propagates to run()'s
+                # top-level handler -> clean whole-batch {"exception": ...}.
+                precondition(ctx)
+            if max_workers <= 1:
+                return _run_sequential(handle_one, jobs, ctx)
+            return _run_parallel(handle_one, jobs, ctx, max_workers)
+
+        return run_batch
+
+    # Support both bare `@per_job` and parameterized `@per_job(max_workers=N)`.
+    if handle is not None:
+        return decorate(handle)
+
+    return decorate
+
+
+def _run_sequential(
+    handle_one: PerJobHandler, jobs: list[Job[Any]], ctx: JobContext[Any]
+) -> list[Any]:
+    results = []
+    for job in jobs:
+        results.append(_run_one(handle_one, job, ctx))
+        ctx.heartbeat()
+    return results
+
+
+def _run_parallel(
+    handle_one: PerJobHandler,
+    jobs: list[Job[Any]],
+    ctx: JobContext[Any],
+    max_workers: int,
+) -> list[Any]:
+    results: list[Any] = [None] * len(jobs)
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        future_to_index = {
+            executor.submit(_run_one, handle_one, job, ctx): index for index, job in enumerate(jobs)
+        }
+        for future in as_completed(future_to_index):
+            index = future_to_index[future]
+            results[index] = future.result()  # _run_one never raises
+            ctx.heartbeat()
+    return results
+
+
+def _run_one(handle_one: PerJobHandler, job: Job[Any], ctx: JobContext[Any]) -> Any:
+    try:
+        return handle_one(job, ctx)
+    except JobException as ex:
+        # A known, expected failure -- report just the message, no traceback noise.
+        return AtmException(exception=str(ex))
+    except Exception:
+        # An unexpected bug -- keep the full traceback for debugging.
+        return AtmException(exception=traceback.format_exc())