taskferry-cloudrun 0.2.0__tar.gz → 0.4.0__tar.gz

taskferry_cloudrun-0.4.0/CHANGELOG.md

@@ -0,0 +1,32 @@
+# Changelog
+
+All notable changes to `taskferry-cloudrun` are documented here.
+The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and
+this project adheres to [Semantic Versioning](https://semver.org/).
+
+## [0.4.0] — 2026-09-19
+
+### Added
+
+- **Native async** submit, state and cancel via the run_v2 async clients
+  (`JobsAsyncClient`, `ExecutionsAsyncClient`). `asubmit`/`aget`/`acancel` use
+  them when available, skipping the worker thread, and fall back to the thread
+  path when the SDK is absent. No new dependency — the async clients ship with
+  `google-cloud-run`.
+
+## [0.3.0] — 2026-09-19
+
+### Added
+
+- **Live log streaming** through Cloud Logging. `stream_logs` pages an execution's
+  log entries in timestamp order, de-duplicating by insert id, and follows them
+  until the execution reaches a terminal state; `logs` returns the whole output
+  once. `logs_uri` (the console link) is unchanged.
+- `CloudRunJobBackend` accepts an injected `logging_client`.
+
+## [0.2.0] — 2026-07-26
+
+### Added
+
+- Initial release, extracted from the Django-coupled backends of `taskferry-django` 0.1
+  and rebuilt against the framework-agnostic Taskferry ports.

{taskferry_cloudrun-0.2.0 → taskferry_cloudrun-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: taskferry-cloudrun
-Version: 0.2.0
+Version: 0.4.0
 Summary: Google Cloud Run Jobs backend for Taskferry — serverless batch workloads.
 Project-URL: Homepage, https://github.com/xiidigital/taskferry
 Project-URL: Documentation, https://taskferry.dev

{taskferry_cloudrun-0.2.0 → taskferry_cloudrun-0.4.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "taskferry-cloudrun"
-version = "0.2.0"
+version = "0.4.0"
 description = "Google Cloud Run Jobs backend for Taskferry — serverless batch workloads."
 readme = "README.md"
 requires-python = ">=3.12"

{taskferry_cloudrun-0.2.0 → taskferry_cloudrun-0.4.0}/src/taskferry_cloudrun/backend.py

@@ -37,6 +37,8 @@ Route GPU work at a backend that really allocates GPUs.
 
 from __future__ import annotations
 
+import time
+from collections.abc import Iterator
 from datetime import UTC, datetime
 from typing import Any
 
@@ -138,6 +140,9 @@ class CloudRunJobBackend(BaseBackend):
         location: str | None = None,
         jobs_client: Any = None,
         executions_client: Any = None,
+        jobs_async_client: Any = None,
+        executions_async_client: Any = None,
+        logging_client: Any = None,
         name: str = "cloudrun",
         tracked_ids: int = 10_000,
     ) -> None:
@@ -150,6 +155,9 @@ class CloudRunJobBackend(BaseBackend):
         self._location = location
         self._jobs_client = jobs_client
         self._executions_client = executions_client
+        self._jobs_async_client = jobs_async_client
+        self._executions_async_client = executions_async_client
+        self._logging_client = logging_client
         self._name = name
         # Cloud Run names executions itself, so remember which of its names goes
         # with which Taskferry id; a full resource path is accepted directly.
@@ -186,6 +194,23 @@ class CloudRunJobBackend(BaseBackend):
             self._executions_client = _run_v2().ExecutionsClient()
         return self._executions_client
 
+    def _logging(self) -> Any:
+        if self._logging_client is None:
+            self._logging_client = _logging_v2().Client(project=self._project)
+        return self._logging_client
+
+    def _async_jobs(self) -> Any:
+        """The async Jobs client: injected, lazily built, or ``None`` if the SDK
+        is absent (the caller then falls back to the thread path)."""
+        if self._jobs_async_client is None and _has_run_v2():
+            self._jobs_async_client = _run_v2().JobsAsyncClient()
+        return self._jobs_async_client
+
+    def _async_executions(self) -> Any:
+        if self._executions_async_client is None and _has_run_v2():
+            self._executions_async_client = _run_v2().ExecutionsAsyncClient()
+        return self._executions_async_client
+
     # -- resource names ----------------------------------------------------------- #
     def job_resource(self, spec: JobSpec) -> str:
         """Fully-qualified Cloud Run Job resource this spec targets."""
@@ -195,25 +220,28 @@ class CloudRunJobBackend(BaseBackend):
         return f"projects/{self._project}/locations/{self._location}/jobs/{spec.job}"
 
     # -- submission ---------------------------------------------------------------- #
-    def _submit(self, spec: ExecutionSpec) -> Execution:
-        assert isinstance(spec, JobSpec)
+    def _submit_request(self, spec: JobSpec) -> dict[str, Any]:
         request: dict[str, Any] = {"name": self.job_resource(spec)}
         overrides = self._overrides(spec)
         if overrides:
             request["overrides"] = overrides
+        return request
 
+    def _submit(self, spec: ExecutionSpec) -> Execution:
+        assert isinstance(spec, JobSpec)
         try:
-            operation = self._jobs().run_job(request=request)
+            operation = self._jobs().run_job(request=self._submit_request(spec))
         except Exception as exc:
             raise SubmissionError(
                 f"Cloud Run could not start job {spec.job!r} in {self._location}: {exc}",
                 backend=self._name,
             ) from exc
+        return self._execution_from_operation(spec, operation)
 
+    def _execution_from_operation(self, spec: JobSpec, operation: Any) -> Execution:
         external_id = _execution_name(operation)
         execution_id = new_execution_id(ExecutionKind.JOB)
         self._ids.remember(str(execution_id), external_id)
-
         return Execution(
             id=execution_id,
             kind=ExecutionKind.JOB,
@@ -255,7 +283,7 @@ class CloudRunJobBackend(BaseBackend):
         return overrides
 
     # -- observation ------------------------------------------------------------------ #
-    def _get(self, execution_id: ExecutionId) -> Execution:
+    def _resolve_external_id(self, execution_id: ExecutionId) -> str:
         external_id = self._ids.resolve(str(execution_id))
         if external_id is None:
             raise ExecutionNotFound(
@@ -264,17 +292,28 @@ class CloudRunJobBackend(BaseBackend):
                 "from another process",
                 backend=self._name,
             )
+        return external_id
+
+    def _get(self, execution_id: ExecutionId) -> Execution:
+        external_id = self._resolve_external_id(execution_id)
         try:
             remote = self._executions().get_execution(name=external_id)
         except Exception as exc:
-            if _is_not_found(exc):
-                raise ExecutionNotFound(
-                    f"Cloud Run has no execution {external_id!r}", backend=self._name
-                ) from exc
-            raise BackendError(
-                f"Cloud Run could not read execution {external_id!r}: {exc}", backend=self._name
-            ) from exc
+            raise self._read_error(external_id, exc) from exc
+        return self._execution_from_remote(execution_id, external_id, remote)
+
+    def _read_error(self, external_id: str, exc: Exception) -> Exception:
+        if _is_not_found(exc):
+            return ExecutionNotFound(
+                f"Cloud Run has no execution {external_id!r}", backend=self._name
+            )
+        return BackendError(
+            f"Cloud Run could not read execution {external_id!r}: {exc}", backend=self._name
+        )
 
+    def _execution_from_remote(
+        self, execution_id: ExecutionId, external_id: str, remote: Any
+    ) -> Execution:
         state, error = map_execution_state(remote)
         return Execution(
             id=execution_id,
@@ -311,6 +350,60 @@ class CloudRunJobBackend(BaseBackend):
             ) from exc
         return self._get(execution_id)
 
+    # -- native async (run_v2 async clients) -------------------------------------- #
+    async def asubmit(self, spec: ExecutionSpec) -> Execution:
+        """Native async submit via ``JobsAsyncClient``, else the thread path."""
+        client = self._async_jobs()
+        if client is None:
+            return await super().asubmit(spec)
+        assert isinstance(spec, JobSpec)
+        self.validate(spec)
+        self.hooks.before_submit(spec, self.name)
+        try:
+            operation = await client.run_job(request=self._submit_request(spec))
+            execution = self._execution_from_operation(spec, operation)
+        except Exception as exc:
+            error = SubmissionError(
+                f"Cloud Run could not start job {spec.job!r} in {self._location}: {exc}",
+                backend=self._name,
+            )
+            self.hooks.on_submit_error(spec, self.name, error)
+            raise error from exc
+        self.hooks.after_submit(spec, execution)
+        return execution
+
+    async def aget(self, execution_id: ExecutionId | str) -> Execution:
+        """Native async state read via ``ExecutionsAsyncClient``."""
+        client = self._async_executions()
+        if client is None:
+            return await super().aget(execution_id)
+        self.capabilities.require(Capability.STATE)
+        eid = ExecutionId(str(execution_id))
+        external_id = self._resolve_external_id(eid)
+        try:
+            remote = await client.get_execution(name=external_id)
+        except Exception as exc:
+            raise self._read_error(external_id, exc) from exc
+        return self._execution_from_remote(eid, external_id, remote)
+
+    async def acancel(self, execution_id: ExecutionId | str) -> Execution:
+        """Native async cancel via ``ExecutionsAsyncClient``."""
+        client = self._async_executions()
+        if client is None:
+            return await super().acancel(execution_id)
+        self.capabilities.require(Capability.CANCEL)
+        eid = ExecutionId(str(execution_id))
+        external_id = self._resolve_external_id(eid)
+        try:
+            await client.cancel_execution(name=external_id)
+        except Exception as exc:
+            raise BackendError(
+                f"Cloud Run could not cancel execution {external_id!r}: {exc}", backend=self._name
+            ) from exc
+        execution = await self.aget(eid)
+        self.hooks.on_cancel(execution)
+        return execution
+
     def logs_uri(self, external_id: str) -> str:
         """A Cloud Logging console link for an execution. No request is made."""
         return (
@@ -318,6 +411,91 @@ class CloudRunJobBackend(BaseBackend):
             f"?project={self._project}&query=resource.labels.location%3D%22{self._location}%22"
         )
 
+    # -- logs ----------------------------------------------------------------------- #
+    def logs(self, execution_id: ExecutionId) -> str:
+        """Read the execution's Cloud Logging output once. Requires ``Capability.LOGS``."""
+        self.capabilities.require(Capability.LOGS)
+        return "\n".join(self.stream_logs(execution_id, follow=False))
+
+    def stream_logs(
+        self,
+        execution_id: ExecutionId,
+        *,
+        follow: bool = True,
+        poll_interval: float = 2.0,
+        timeout: float | None = None,
+    ) -> Iterator[str]:
+        """Yield the execution's Cloud Logging entries, following them live by default.
+
+        Cloud Logging is queried by filter rather than truly streamed, so this
+        pages entries in timestamp order, de-duplicating by insert id, and — with
+        ``follow=True`` — keeps polling for new entries until the execution reaches
+        a terminal state. With ``follow=False`` it returns what is there and stops.
+        ``timeout`` bounds the total stream. Requires ``Capability.LOGS``.
+        """
+        self.capabilities.require(Capability.LOGS)
+        external_id = self._ids.resolve(str(execution_id))
+        if external_id is None:
+            raise ExecutionNotFound(
+                f"{execution_id!r} was not submitted by this backend instance; pass the "
+                "Cloud Run execution name to stream its logs from another process",
+                backend=self._name,
+            )
+        execution_name = external_id.rsplit("/executions/", 1)[-1]
+        base_filter = (
+            'resource.type="cloud_run_job" '
+            f'AND resource.labels.location="{self._location}" '
+            f'AND labels."run.googleapis.com/execution_name"="{execution_name}"'
+        )
+        deadline = None if timeout is None else time.monotonic() + timeout
+        client = self._logging()
+        seen: set[str] = set()
+        since: str | None = None
+        while True:
+            log_filter = base_filter if since is None else f'{base_filter} AND timestamp>="{since}"'
+            try:
+                entries = list(
+                    client.list_entries(
+                        resource_names=[f"projects/{self._project}"],
+                        filter_=log_filter,
+                        order_by="timestamp asc",
+                        page_size=1000,
+                    )
+                )
+            except Exception as exc:
+                if _is_not_found(exc):
+                    return
+                raise BackendError(
+                    f"Cloud Run could not read logs for {external_id!r}: {exc}", backend=self._name
+                ) from exc
+            new = 0
+            for entry in entries:
+                insert_id = str(getattr(entry, "insert_id", "") or "")
+                if insert_id and insert_id in seen:
+                    continue
+                if insert_id:
+                    seen.add(insert_id)
+                timestamp = getattr(entry, "timestamp", None)
+                if timestamp is not None:
+                    since = _rfc3339(timestamp)
+                text = _entry_text(entry)
+                if text is not None:
+                    yield text
+                    new += 1
+            if new == 0:
+                if not follow or self._is_terminal(external_id):
+                    return
+                if deadline is not None and time.monotonic() > deadline:
+                    return
+                time.sleep(poll_interval)
+
+    def _is_terminal(self, external_id: str) -> bool:
+        try:
+            remote = self._executions().get_execution(name=external_id)
+        except Exception:  # a vanished execution is, for our purposes, done
+            return True
+        return map_execution_state(remote)[0].is_terminal
+
 
 def _run_v2() -> Any:
     """Import ``google.cloud.run_v2`` lazily, with an actionable error."""
@@ -330,6 +508,49 @@ def _run_v2() -> Any:
     return run_v2
 
 
+def _has_run_v2() -> bool:
+    """Whether the Cloud Run SDK is importable, without importing it."""
+    from importlib.util import find_spec
+
+    try:
+        return find_spec("google.cloud.run_v2") is not None
+    except ModuleNotFoundError:
+        return False
+
+
+def _logging_v2() -> Any:
+    """Import ``google.cloud.logging_v2`` lazily, with an actionable error."""
+    try:
+        from google.cloud import logging_v2
+    except ImportError as exc:  # pragma: no cover - depends on the environment
+        raise ConfigurationError(
+            "the Cloud Run backend needs the Google logging SDK: "
+            "pip install 'taskferry-cloudrun[gcp]'"
+        ) from exc
+    return logging_v2
+
+
+def _entry_text(entry: Any) -> str | None:
+    """Extract a line of text from a Cloud Logging entry (text or struct payload)."""
+    payload = getattr(entry, "payload", None)
+    if payload is None:
+        return None
+    if isinstance(payload, str):
+        return payload.rstrip("\n")
+    if isinstance(payload, dict):
+        message = payload.get("message")
+        return str(message if message is not None else payload).rstrip("\n")
+    return str(payload).rstrip("\n")
+
+
+def _rfc3339(timestamp: Any) -> str:
+    """Format a timestamp for a Cloud Logging ``timestamp>=`` filter."""
+    if isinstance(timestamp, datetime):
+        moment = timestamp if timestamp.tzinfo else timestamp.replace(tzinfo=UTC)
+        return moment.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+    return str(timestamp)
+
+
 def _execution_name(operation: Any) -> str | None:
     """Pull the execution resource name out of the long-running operation."""
     metadata = getattr(operation, "metadata", None)
@@ -373,6 +594,9 @@ def make_backend(**options: Any) -> CloudRunJobBackend:
         location=options.get("location"),
         jobs_client=options.get("jobs_client"),
         executions_client=options.get("executions_client"),
+        jobs_async_client=options.get("jobs_async_client"),
+        executions_async_client=options.get("executions_async_client"),
+        logging_client=options.get("logging_client"),
         name=str(options.get("name", "cloudrun")),
         tracked_ids=int(options.get("tracked_ids", 10_000)),
     )

taskferry_cloudrun-0.2.0/CHANGELOG.md

@@ -1,12 +0,0 @@
-# Changelog
-
-All notable changes to `taskferry-cloudrun` are documented here.
-The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and
-this project adheres to [Semantic Versioning](https://semver.org/).
-
-## [0.2.0] — 2026-07-26
-
-### Added
-
-- Initial release, extracted from the Django-coupled backends of `taskferry-django` 0.1
-  and rebuilt against the framework-agnostic Taskferry ports.