onedata-lambda-sdk 1.0.0__py3-none-any.whl

onedata_lambda_sdk/runtime.py

@@ -0,0 +1,388 @@
+"""
+The lambda runtime contract: `run()` -- the single entry point the base image's
+bootstrap calls (`from onedata_lambda_sdk import run; run(handler, raw)`).
+
+The logic is migrated from the v2 base `index.py` (which ran as a fresh process per
+request); here it lives in the SDK and is driven in-process. The `@per_job` adapter that
+produces a batch handler for `run()` lives in `perjob`.
+"""
+
+__author__ = "Bartosz Walkowicz"
+__copyright__ = "Copyright (C) 2026 Onedata (onedata.org)"
+__license__ = "This software is released under the MIT license cited in LICENSE.txt"
+
+import json
+import os
+import sys
+import threading
+import time
+import traceback
+import uuid
+from collections.abc import Callable, Iterator
+from contextlib import contextmanager
+from datetime import datetime
+from typing import Any, Final
+
+import requests
+
+from ._wire import AtmJobArgs, AtmJobBatchRequest, AtmJobBatchRequestCtx, AtmJobBatchResponse
+from .job import Job, JobContext, JobException
+from .logging import DEFAULT_LOG_LEVEL, Logger
+from .oneclient import ENV_MOUNT_POINT
+from .streaming import ResultStreamer, StreamFlusher
+from .types import AtmException
+
+
+# --- Environment contract --------------------------------------------------------------
+# Every environment variable this module reads, named in one place so the runtime's
+# dependency on the environment is visible at a glance. Read via `_get_env` at call time
+# (not import time) so tests and dev-mode can set them per run.
+ENV_VERIFY_SSL: Final[str] = "VERIFY_SSL_CERTIFICATES"
+ENV_DEBUG_MODE: Final[str] = "OPENFAAS_FUNCTION_DEBUG_MODE"
+ENV_ONECLIENT_MOUNTED: Final[str] = "ONECLIENT_MOUNTED"
+ENV_DEV_MODE: Final[str] = "ONEDATA_LAMBDA_DEV"
+# The mount-point env name is owned by `oneclient` (one source of truth shared with the
+# `mounted_file_path` helper handlers use), re-exported here for the mount-wait below.
+
+# Local file the runtime's diagnostics (and any stray handler prints) are redirected to,
+# so they never reach stdout/stderr (which OpenFaaS reads as the response).
+LAMBDA_OUTPUT_LOG: Final[str] = "/tmp/lambda.out"
+
+HEARTBEAT_TIMEOUT_SEC: Final[int] = 10
+FIRST_HEARTBEAT_RETRIES: Final[int] = 10
+FIRST_HEARTBEAT_RETRY_DELAY_SEC: Final[int] = 5
+
+MOUNT_POINT_TIMEOUT_SEC: Final[int] = 120
+MOUNT_POINT_POLL_INTERVAL_SEC: Final[float] = 0.5
+
+#: A runtime diagnostic logger: `log(severity, message)` -> /tmp/lambda.out.
+DiagLogger = Callable[[str, str], None]
+
+
+class _FirstHeartbeatError(Exception):
+    pass
+
+
+class _OneclientMountError(Exception):
+    pass
+
+
+def run(
+    handler: Callable[..., Any], raw: str, *, out_dir: str | None = None
+) -> AtmJobBatchResponse[Any] | AtmException:
+    """
+    Run a batch-level `handle(jobs, ctx)` handler against a raw OpenFaaS request and
+    return the response envelope as a dict.
+
+    Owns everything between the raw request and the handler:
+
+    * parse the request into the per-batch `JobContext` and the `Job` list;
+    * redirect stdout/stderr to LAMBDA_OUTPUT_LOG so stray handler prints don't corrupt the
+      response, and write the runtime's own lifecycle diagnostics there;
+    * deliver the mandatory first heartbeat, then vend a throttled `heartbeat()` service;
+    * wait for the Oneclient mount when the lambda uses one;
+    * dispatch the handler and assemble the envelope.
+
+    Never raises for an expected failure: a whole-batch handler error (or a malformed
+    request) becomes a top-level `{"exception": ...}`; per-job failures are the
+    `AtmException` entries the handler placed in its `{"resultsBatch": ...}`.
+
+    `out_dir` overrides where result/log streams are written (default `/out`); production
+    leaves it unset, tests inject a temp dir (see `testing.run_local`).
+    """
+    batch_id = uuid.uuid4()
+    debug_mode = _get_env(ENV_DEBUG_MODE) == "true"
+
+    with _redirected_streams():
+
+        def log(severity: str, message: str) -> None:
+            _diag_log(batch_id, severity, message)
+
+        log("info", "Started request processing")
+        if debug_mode:
+            log("debug", "Lambda is running in debug mode")
+
+        try:
+            _await_oneclient_mount(log)
+
+            # Shape assertion, not validation: json.loads returns Any, and malformed input
+            # surfaces as the top-level exception envelope below rather than a type error.
+            request: AtmJobBatchRequest[Any] = json.loads(raw)
+            if debug_mode:
+                log("debug", f"Input: {request}")
+
+            ctx_wire = request["ctx"]
+
+            send_heartbeat = _make_heartbeat_sender(ctx_wire, log)
+            # The first heartbeat is mandatory and unconditional: it tells Oneprovider
+            # the job batch actually started executing.
+            log("info", "Sending first heartbeat")
+            _deliver_first_heartbeat(send_heartbeat)
+
+            # Narrow the internal sender to the public heartbeat service: the inward API
+            # (JobContext.heartbeat, the streamers, the flusher) is typed `Callable[[], None]`,
+            # while `send_heartbeat` is `(*, force: bool) -> bool`. This adapter hides the
+            # `force=` throttle-bypass (runtime-only, for the first heartbeat) and the success
+            # bool, so neither leaks into the inward-facing contract.
+            def heartbeat() -> None:
+                send_heartbeat()
+
+            flusher = StreamFlusher(heartbeat=heartbeat, out_dir=out_dir)
+            try:
+                jobs, job_ctx = _build_jobs_and_context(request, heartbeat, flusher, out_dir)
+
+                log("info", "Calling handler callback")
+                results = handler(jobs, job_ctx)
+                response: AtmJobBatchResponse[Any] = {"resultsBatch": results}
+                if debug_mode:
+                    log("debug", f"Output: {response}")
+                log("info", "Ended request processing")
+
+                return response
+            finally:
+                # Drain buffered streams/logs before responding, even on error.
+                flusher.stop()
+        except _OneclientMountError:
+            log("error", "Request processing failed to mount Oneclient")
+            return AtmException(exception="Failed to mount Oneclient")
+        except _FirstHeartbeatError:
+            log("error", "Request processing failed to deliver first heartbeat")
+            return AtmException(exception="Failed to deliver first heartbeat")
+        except JobException as ex:
+            # A known, whole-batch failure raised by a (batch-style) handler -- report
+            # just the message, no traceback noise. Per-job failures never reach here:
+            # `@per_job` turns them into AtmException entries inside the results batch.
+            log("error", f"Request processing failed: {ex}")
+            return AtmException(exception=str(ex))
+        except Exception as ex:
+            log("error", f"Request processing failed with unexpected exception: {ex}")
+            return AtmException(exception=traceback.format_exc())
+
+
+@contextmanager
+def _redirected_streams() -> Iterator[None]:
+    """
+    Redirect fd 1/2 to LAMBDA_OUTPUT_LOG for the duration of the call.
+
+    OpenFaaS treats anything written to the function's stdout as (part of) the response,
+    so accidental `print`s from a handler -- or a library it uses -- would corrupt it.
+    We redirect at the file-descriptor level (not just `sys.stdout`) so writes from C
+    extensions and subprocesses are captured too, and restore on the way out so the
+    bootstrap can write the real response. The runtime's own diagnostics go to stderr,
+    which is redirected here too, so they land in the same file.
+    """
+    if _is_dev_mode():
+        # Keep stdout/stderr so handler output and diagnostics are visible locally.
+        yield
+        return
+
+    sink = open(LAMBDA_OUTPUT_LOG, "a")  # noqa: SIM115 -- manual fd lifecycle (restored in finally)
+    saved_stdout_fd = os.dup(sys.stdout.fileno())
+    saved_stderr_fd = os.dup(sys.stderr.fileno())
+    try:
+        os.dup2(sink.fileno(), sys.stdout.fileno())  # $ exec >&tmp_out_fd
+        os.dup2(sink.fileno(), sys.stderr.fileno())  # $ exec 2>&tmp_out_fd
+        yield
+    finally:
+        sys.stdout.flush()
+        sys.stderr.flush()
+        os.dup2(saved_stdout_fd, sys.stdout.fileno())
+        os.dup2(saved_stderr_fd, sys.stderr.fileno())
+        # `os.dup` above allocated two fresh descriptors; once dup2 has restored fd 1/2 to
+        # point at them, the saved copies are redundant. Close them (and the sink) or we
+        # leak two fds per request -- and run() is called once per request in a long-lived
+        # process, so the leak would accumulate until the fd limit is hit. Closing the
+        # copies does not disturb the just-restored fd 1/2 (independent descriptors).
+        os.close(saved_stdout_fd)
+        os.close(saved_stderr_fd)
+        sink.close()
+
+
+def _diag_log(job_id: uuid.UUID, severity: str, message: str) -> None:
+    # The runtime's own lifecycle trace. In production stdout/stderr are redirected to
+    # LAMBDA_OUTPUT_LOG (see _redirected_streams), so this lands there; in dev mode it
+    # stays on the console. Distinct from the audit-log stream a handler writes via
+    # ctx.logger(name).
+    timestamp = datetime.now().strftime("%m/%d/%Y, %H:%M:%S:%f")
+    print(f"[{severity.upper()} {timestamp} {job_id}] {message}", file=sys.stderr)
+
+
+def _await_oneclient_mount(log: DiagLogger) -> None:
+    if _is_dev_mode():
+        log("debug", "Skipping Oneclient mount wait (dev mode)")
+        return
+
+    if _get_env(ENV_ONECLIENT_MOUNTED) != "true":
+        log("info", "This lambda is not using Oneclient")
+        return
+
+    mount_point = _get_env(ENV_MOUNT_POINT)
+    if not mount_point:
+        log("error", "Oneclient declared mounted but no mount point given")
+        raise _OneclientMountError
+
+    log("info", f"This lambda is using Oneclient; mount point: {mount_point}")
+    deadline = time.monotonic() + MOUNT_POINT_TIMEOUT_SEC
+    while time.monotonic() < deadline:
+        if any(os.listdir(mount_point)):
+            log("info", f"Oneclient mounted: {os.listdir(mount_point)}")
+            return
+        time.sleep(MOUNT_POINT_POLL_INTERVAL_SEC)
+
+    log("error", "Did not manage to mount Oneclient")
+    raise _OneclientMountError
+
+
+def _make_heartbeat_sender(
+    ctx_wire: AtmJobBatchRequestCtx[Any], log: DiagLogger
+) -> Callable[..., bool]:
+    """
+    Build a throttled heartbeat sender returning whether a recent heartbeat is in place.
+
+    Throttled to one POST per `0.1 * timeoutSeconds` (a call within that window is a
+    no-op reported as success, since a fresh heartbeat already exists). `force=True`
+    bypasses the throttle (used for the mandatory first heartbeat).
+    """
+    if _is_dev_mode():
+
+        def log_heartbeat(*, force: bool = False) -> bool:
+            log("debug", f"Heartbeat (force={force}) [dev mode, not sent]")
+            return True
+
+        return log_heartbeat
+
+    url = ctx_wire["heartbeatUrl"]
+    throttle_period = 0.1 * ctx_wire["timeoutSeconds"]
+    verify = _get_env(ENV_VERIFY_SSL) == "true"
+    last_sent_at: float | None = None
+    # Guards last_sent_at so parallel `@per_job` workers don't tear it. The POST runs
+    # outside the lock (no serializing the network); the worst a race can do is send one
+    # extra heartbeat, which is harmless.
+    lock = threading.Lock()
+
+    def send(*, force: bool = False) -> bool:
+        nonlocal last_sent_at
+        now = time.monotonic()
+        with lock:
+            recent = last_sent_at is not None and now - last_sent_at < throttle_period
+
+        if not force and recent:
+            return True
+
+        try:
+            response = requests.post(url, data={}, timeout=HEARTBEAT_TIMEOUT_SEC, verify=verify)
+        except requests.RequestException as ex:
+            log("warning", f"Failed to send heartbeat: {ex}")
+            return False
+
+        if response.ok:
+            with lock:
+                last_sent_at = now
+            log("info", "Heartbeat sent")
+            return True
+
+        log(
+            "warning",
+            f"Failed to send heartbeat due to:\n"
+            f"> response code: {response.status_code}\n"
+            f"> response content: {response.content!r}",
+        )
+        return False
+
+    return send
+
+
+def _deliver_first_heartbeat(send_heartbeat: Callable[..., bool]) -> None:
+    for _ in range(FIRST_HEARTBEAT_RETRIES):
+        if send_heartbeat(force=True):
+            return
+        time.sleep(FIRST_HEARTBEAT_RETRY_DELAY_SEC)
+    raise _FirstHeartbeatError
+
+
+def _build_jobs_and_context(
+    request: AtmJobBatchRequest[Any],
+    heartbeat: Callable[[], None],
+    flusher: StreamFlusher,
+    out_dir: str | None,
+) -> tuple[list[Job[Any]], JobContext[Any]]:
+    ctx_wire = request["ctx"]
+    log_level = ctx_wire.get("logLevel", DEFAULT_LOG_LEVEL)
+    jobs = [_build_job(item) for item in request["argsBatch"]]
+
+    # One streamer/logger instance per stream name, shared across the batch's jobs (they
+    # append to the same /out/<name> through the shared flusher).
+    streamers: dict[str, ResultStreamer[Any]] = {}
+
+    def result_streamer_factory(name: str, *, buffered: bool = True) -> ResultStreamer[Any]:
+        streamer = streamers.get(name)
+        if streamer is None:
+            streamer = ResultStreamer(
+                result_name=name,
+                buffered=buffered,
+                flusher=flusher,
+                heartbeat=heartbeat,
+                out_dir=out_dir,
+            )
+            streamers[name] = streamer
+        return streamer
+
+    loggers: dict[str, Logger] = {}
+
+    def logger_factory(name: str) -> Logger:
+        # Logs are low-frequency and worth keeping even on a crash -> direct (unbuffered).
+        logger = loggers.get(name)
+        if logger is None:
+            logger = Logger(
+                result_name=name,
+                buffered=False,
+                flusher=flusher,
+                heartbeat=heartbeat,
+                log_level=log_level,
+                out_dir=out_dir,
+            )
+            loggers[name] = logger
+        return logger
+
+    job_ctx: JobContext[Any] = JobContext(
+        config=ctx_wire.get("config"),
+        user_id=ctx_wire["userId"],
+        space_id=ctx_wire["spaceId"],
+        workflow_execution_id=ctx_wire["atmWorkflowExecutionId"],
+        oneprovider_id=ctx_wire["oneproviderId"],
+        oneprovider_domain=ctx_wire["oneproviderDomain"],
+        onezone_domain=ctx_wire["onezoneDomain"],
+        access_token=ctx_wire["accessToken"],
+        timeout_seconds=ctx_wire["timeoutSeconds"],
+        log_level=log_level,
+        heartbeat=heartbeat,
+        result_streamer_factory=result_streamer_factory,
+        logger_factory=logger_factory,
+        raw=ctx_wire,
+    )
+    return jobs, job_ctx
+
+
+def _build_job(item: AtmJobArgs) -> Job[Any]:
+    # trace_id comes from the wire: the framework assigns it per job in a `__meta`
+    # envelope. We never fabricate it -- a missing __meta.traceId is a malformed request
+    # (top-level exception), so contract drift surfaces instead of being masked. The
+    # envelope is stripped so the handler sees only its declared arguments.
+    try:
+        trace_id = item["__meta"]["traceId"]
+    except (KeyError, TypeError) as ex:
+        raise RuntimeError(f"job args missing __meta.traceId: {ex}") from ex
+
+    args = {key: value for key, value in item.items() if key != "__meta"}
+    return Job(args=args, trace_id=trace_id)
+
+
+def _get_env(name: str, default: str | None = None) -> str | None:
+    return os.environ.get(name, default)
+
+
+def _is_dev_mode() -> bool:
+    # Local dev / runtime-test mode (ONEDATA_LAMBDA_DEV=1): log heartbeats instead of
+    # POSTing them, skip the stdout/stderr redirect, and skip the Oneclient mount-wait,
+    # so the real runtime can be driven locally on a fixture. See the `testing` module.
+    return _get_env(ENV_DEV_MODE) == "1"

onedata_lambda_sdk/stats.py

@@ -0,0 +1,53 @@
+"""Time-series measurements for lambdas."""
+
+__author__ = "Bartosz Walkowicz"
+__copyright__ = "Copyright (C) 2022-2026 Onedata (onedata.org)"
+__license__ = "This software is released under the MIT license cited in LICENSE.txt"
+
+
+import time
+
+from .types import AtmTimeSeriesMeasurement
+
+
+class TimeSeriesMeasurementBuilder:
+    """Base class for time series measurement builders.
+
+    In order to create a builder for concrete time series measurement, create
+    a class deriving from TimeSeriesMeasurementBuilder with following
+    metadata specified:
+    - ts_name - used as exact name of the measurement
+    - unit - stored only as metadata and not processed in any way
+
+    Example usage:
+    >>>
+    >>> import pprint
+    >>>
+    >>> from onedata_lambda_sdk import TimeSeriesMeasurementBuilder
+    >>>
+    >>>
+    >>> class FilesProcessed(
+    >>>     TimeSeriesMeasurementBuilder, ts_name="filesProcessed", unit=None
+    >>> ):
+    >>>     pass
+    >>>
+    >>>
+    >>> pprint.pprint(FilesProcessed.build(value=34, timestamp=100))
+
+         {'timestamp': 100, 'tsName': 'filesProcessed', 'value': 34}
+    """
+
+    _ts_name: str
+    _unit: str | None
+
+    def __init_subclass__(cls, ts_name: str, unit: str | None) -> None:
+        cls._ts_name = ts_name
+        cls._unit = unit
+
+    @classmethod
+    def build(cls, value: float, *, timestamp: int | None = None) -> AtmTimeSeriesMeasurement:
+        return {
+            "tsName": cls._ts_name,
+            "timestamp": int(time.time()) if timestamp is None else timestamp,
+            "value": value,
+        }

onedata_lambda_sdk/streaming.py

@@ -0,0 +1,158 @@
+"""
+Streaming lambda results and measurements to `/out/<name>`.
+
+Beyond the per-job results a handler returns, a lambda can append items incrementally to
+named streams (audit logs, time-series measurements, large result sets); this module
+owns that path. The runtime owns the flusher's lifecycle -- lazily started on first
+buffered use and drained-and-stopped when the handler returns (see `runtime.run`).
+Everything here is thread-safe.
+"""
+
+__author__ = "Bartosz Walkowicz"
+__copyright__ = "Copyright (C) 2022-2026 Onedata (onedata.org)"
+__license__ = "This software is released under the MIT license cited in LICENSE.txt"
+
+import json
+import os
+import threading
+from collections.abc import Callable, Iterable
+from typing import Any, Final
+
+
+OUT_DIR: Final[str] = "/out"
+
+# How often the background flusher drains buffered items.
+FLUSH_INTERVAL_SEC: Final[float] = 1.0
+
+
+class StreamFlusher:
+    """
+    A single background thread batching writes for all buffered streams of one run.
+
+    Producers `enqueue` `(stream_name, item)` pairs (cheap, lock-guarded). The
+    thread wakes periodically, writes everything buffered grouped by stream, and -- only
+    if it actually wrote something -- emits a heartbeat (itself throttled by the
+    runtime). The thread starts lazily on first `enqueue` so idle lambdas pay nothing.
+    """
+
+    def __init__(
+        self,
+        *,
+        heartbeat: Callable[[], None],
+        out_dir: str | None = None,
+        interval: float = FLUSH_INTERVAL_SEC,
+    ) -> None:
+        self._heartbeat = heartbeat
+        self._out_dir = out_dir if out_dir is not None else OUT_DIR
+        self._interval = interval
+        self._buffer: list[tuple[str, Any]] = []
+        self._buffer_lock = threading.Lock()
+        self._start_lock = threading.Lock()
+        self._stop = threading.Event()
+        self._thread: threading.Thread | None = None
+
+    def enqueue(self, name: str, item: Any) -> None:
+        self.enqueue_many(name, [item])
+
+    def enqueue_many(self, name: str, items: list[Any]) -> None:
+        if not items:
+            return
+
+        self._ensure_started()
+        with self._buffer_lock:
+            self._buffer.extend((name, item) for item in items)
+
+    def _ensure_started(self) -> None:
+        if self._thread is not None:
+            return
+
+        with self._start_lock:
+            if self._thread is None:
+                thread = threading.Thread(target=self._loop, daemon=True)
+                self._thread = thread
+                thread.start()
+
+    def _loop(self) -> None:
+        while not self._stop.is_set():
+            self._stop.wait(self._interval)
+            self._flush_once()
+
+    def _flush_once(self) -> None:
+        with self._buffer_lock:
+            if not self._buffer:
+                return
+
+            pending = self._buffer
+            self._buffer = []
+
+        by_name: dict[str, list[Any]] = {}
+        for name, item in pending:
+            by_name.setdefault(name, []).append(item)
+        for name, items in by_name.items():
+            _append_to_stream(self._out_dir, name, items)
+
+        # Activity-gated: heartbeat only because we made observable progress.
+        self._heartbeat()
+
+    def stop(self) -> None:
+        """Stop the thread and drain remaining buffered items (no-op if unused)."""
+        self._stop.set()
+        if self._thread is not None:
+            self._thread.join()
+        self._flush_once()
+
+
+class ResultStreamer[T]:
+    """
+    Streams items to `/out/<result_name>`. Two modes, neither needing a manual flush:
+
+    * **buffered (default)** -- enqueues into the shared background flusher, which batches
+      writes across all buffered streams and heartbeats after each non-empty flush. Best
+      for high-frequency / multi-threaded producers.
+    * **direct** (`buffered=False`) -- writes immediately and heartbeats (throttled) per
+      write. Best for low-frequency output or callers doing their own batching.
+
+    Vended by `JobContext.result_streamer(name, *, buffered=...)`; the runtime injects
+    the shared flusher and the heartbeat. Construct directly only in tests.
+    """
+
+    def __init__(
+        self,
+        *,
+        result_name: str,
+        buffered: bool,
+        flusher: StreamFlusher,
+        heartbeat: Callable[[], None],
+        out_dir: str | None = None,
+    ) -> None:
+        self.result_name = result_name
+        self._buffered = buffered
+        self._flusher = flusher
+        self._heartbeat = heartbeat
+        self._out_dir = out_dir if out_dir is not None else OUT_DIR
+        self._write_lock = threading.Lock()
+
+    def stream_item(self, item: T) -> None:
+        self.stream_items([item])
+
+    def stream_items(self, items: Iterable[T]) -> None:
+        materialized = list(items)
+        if not materialized:
+            return
+
+        if self._buffered:
+            # One lock acquisition for the whole batch, not one per item.
+            self._flusher.enqueue_many(self.result_name, materialized)
+        else:
+            with self._write_lock:
+                _append_to_stream(self._out_dir, self.result_name, materialized)
+
+            # Direct mode: heartbeat per write (throttled by the runtime).
+            self._heartbeat()
+
+
+def _append_to_stream(out_dir: str, name: str, items: list[Any]) -> None:
+    with open(os.path.join(out_dir, name), "a") as stream_file:
+        for item in items:
+            json.dump(item, stream_file)
+            stream_file.write("\n")