flyteplugins-union 0.5.0__tar.gz → 0.5.2__tar.gz

{flyteplugins_union-0.5.0/src/flyteplugins_union.egg-info → flyteplugins_union-0.5.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flyteplugins-union
-Version: 0.5.0
+Version: 0.5.2
 Summary: Union SDK - Proprietary extensions for Flyte
 Author-email: unionai <info@union.ai>
 Project-URL: Homepage, https://www.union.ai/

{flyteplugins_union-0.5.0 → flyteplugins_union-0.5.2}/licenses/BSL.txt

@@ -17,7 +17,7 @@ Additional Use Grant: You may make use of the Licensed Work, provided that
                       computer programs whose source code are controlled by
                       such third parties.
 
-Change Date:          2030-06-29
+Change Date:          2030-07-13
 
 Change License:       Apache License, Version 2.0
 

{flyteplugins_union-0.5.0 → flyteplugins_union-0.5.2}/src/flyteplugins/union/cli/role.py

@@ -21,10 +21,13 @@ def _new_role_yaml(name: str, description: str = "") -> str:
 
 
 def _role_to_yaml(role: Role) -> str:
-    """Serialize a Role to YAML for interactive editing."""
-    data = {"name": role.name, "description": role.description, "actions": role.actions}
+    """Serialize a Role to YAML for interactive editing.
+
+    Only actions are editable; the role name and description cannot be changed.
+    """
+    data = {"name": role.name, "actions": role.actions}
     body = _yaml.safe_dump(data, default_flow_style=False, sort_keys=False)
-    comment = "# Valid actions:\n"
+    comment = "# Only 'actions' can be edited. Valid actions:\n"
     for a in _ALL_ACTIONS:
         comment += f"#   - {a}\n"
     return comment + "\n" + body
@@ -141,9 +144,10 @@ def delete_role(cfg: common.CLIConfig, name: str, yes: bool):
 @click.argument("name", type=str)
 @click.pass_obj
 def update_role(cfg: common.CLIConfig, name: str):
-    """Update a role interactively.
+    """Update a role's actions interactively.
 
     Opens the role in your $EDITOR as YAML. Save and close to apply changes.
+    Only actions can be edited; the role name and description cannot be changed.
 
     Examples:
 
@@ -161,11 +165,13 @@ def update_role(cfg: common.CLIConfig, name: str):
 
     def _apply(edited_yaml):
         data = _yaml.safe_load(edited_yaml)
-        new_description = data.get("description", "")
         new_actions = sorted(data.get("actions") or [])
-        if new_description == role.description and new_actions == sorted(role.actions):
+        if new_actions == sorted(role.actions):
             return "no_changes"
-        Role.update(name, description=new_description, actions=data.get("actions"))
+        Role.update(name, actions=data.get("actions") or [])
+        # edit_with_retry treats None as "cancelled"; return a marker so the
+        # success message below is printed.
+        return "updated"
 
     result = edit_with_retry(yaml_text, _apply, console=console, noun="role")
     if result == "no_changes":

{flyteplugins_union-0.5.0 → flyteplugins_union-0.5.2}/src/flyteplugins/union/io/_base_volume.py

@@ -56,6 +56,7 @@ automatically at task return) also unmounts.
 from __future__ import annotations
 
 import asyncio
+import functools
 import os
 import re
 import secrets
@@ -172,6 +173,30 @@ def _ensure_etc_mtab() -> None:
 _VOLUME_BACKENDS: dict[str, _VolumeBackend] = {"juicefs": JuiceFSVolumeBackend()}
 
 
+def _lifecycle_serialized(method):
+    """Serialize lifecycle ops (mount / fork / commit) per metadata store.
+
+    A background checkpoint loop (:meth:`RWVolume.commit` on a timer) can
+    otherwise interleave with a concurrent :meth:`fork` or terminal
+    :meth:`RWVolume.finalize` on the same volume: fork's live path unmounts
+    and remounts the store mid-snapshot, so an unserialized checkpoint can
+    flush a store that fork just tore down, snapshot a half-materialized
+    index, or double-register the mount daemon. All entry points funnel
+    through one ``asyncio.Lock`` keyed by resolved ``meta_dir`` (the store
+    identity; mount paths alias it 1:1 today). Internal helpers stay
+    undecorated — only entry points take the lock, so the
+    ``_commit_unmount`` → keep-alive fallback can't self-deadlock.
+    """
+
+    @functools.wraps(method)
+    async def wrapper(self, *args, **kwargs):
+        meta_dir = kwargs.get("meta_dir") or self._active_meta_dir or self._default_meta_dir()
+        async with type(self)._lifecycle_lock(meta_dir):
+            return await method(self, *args, **kwargs)
+
+    return wrapper
+
+
 class ActionRef(BaseModel):
     """Provenance: the action (one task execution within a run) that
     produced a particular :class:`Volume` version, plus the output slot it
@@ -312,6 +337,13 @@ class Volume(BaseModel):
     # to decide whether to use the live flush-and-upload path or the cold
     # File.copy_to path. Populated by mount(), cleared by commit().
     _live_meta: ClassVar[Set[str]] = set()
+    # One lock per metadata store, taken by every lifecycle entry point via
+    # ``@_lifecycle_serialized``. Never handed out — see that decorator.
+    _lifecycle_locks: ClassVar[Dict[str, asyncio.Lock]] = {}
+
+    @classmethod
+    def _lifecycle_lock(cls, meta_dir: str) -> asyncio.Lock:
+        return cls._lifecycle_locks.setdefault(meta_dir, asyncio.Lock())
 
     # Per-instance: where THIS Volume was last mounted by mount(), so the
     # type transformer can auto-commit on task return without the caller
@@ -435,6 +467,7 @@ class Volume(BaseModel):
         base = cls.empty(name, bucket, storage=storage, region=region, metadata_store_type=metadata_store_type)
         return _coerce_class(base, RWVolume)
 
+    @_lifecycle_serialized
     async def mount(
         self,
         *,
@@ -774,6 +807,7 @@ class Volume(BaseModel):
             message=message,
         )
 
+    @_lifecycle_serialized
     async def _commit_unmount(
         self,
         *,
@@ -840,7 +874,8 @@ class Volume(BaseModel):
                 "add `RUN ln -sf /proc/self/mounts /etc/mtab` to the task image.",
                 e,
             )
-            return await self._commit_keepalive(mount_path=mount_path, meta_dir=meta_dir, message=message)
+            # Direct impl call: we already hold this store's lifecycle lock.
+            return await self._commit_keepalive_impl(mount_path=mount_path, meta_dir=meta_dir, message=message)
         new_index, index_bytes, metadata_path = await self._snapshot_and_upload_index(
             meta_dir=meta_dir, tmp_prefix="vol-commit-", flush_live=False
         )
@@ -1016,6 +1051,7 @@ class Volume(BaseModel):
             except OSError:
                 pass
 
+    @_lifecycle_serialized
     async def fork(
         self,
         name: str,
@@ -1195,12 +1231,25 @@ class Volume(BaseModel):
 
         return config
 
+    @_lifecycle_serialized
     async def _commit_keepalive(
         self,
         *,
         mount_path: Optional[str] = None,
         meta_dir: Optional[str] = None,
         message: Optional[str] = None,
+    ) -> "Volume":
+        """Serialized entry point for keep-alive commits (see
+        :func:`_lifecycle_serialized`); :meth:`_commit_unmount` calls the impl
+        directly because it already holds the store's lock."""
+        return await self._commit_keepalive_impl(mount_path=mount_path, meta_dir=meta_dir, message=message)
+
+    async def _commit_keepalive_impl(
+        self,
+        *,
+        mount_path: Optional[str] = None,
+        meta_dir: Optional[str] = None,
+        message: Optional[str] = None,
     ) -> "Volume":
         """Drain writeback, then snapshot + publish the live index — without
         unmounting. The keep-alive counterpart to :meth:`_commit_unmount`,

{flyteplugins_union-0.5.0 → flyteplugins_union-0.5.2}/src/flyteplugins/union/io/_internal/_juicefs/_backend.py

@@ -318,6 +318,20 @@ class JuiceFSVolumeBackend:
         cmd = [
             self.client_binary,
             "mount",
+            # Branches share one bucket but each has a private metadata DB, so a
+            # client's slice refcounts can't see sibling/parent references. Any
+            # client-side object deletion (slice compaction — fired inline on
+            # the read/write path, not bgjob-gated — file purge, trash expiry)
+            # may therefore destroy chunks other branches still reference.
+            # --max-deletes 0 no-ops the single gate all those paths funnel
+            # through (meta.deleteSlice); compaction still rewrites metadata
+            # and writes the merged slice, it just never deletes old objects.
+            # --no-bgjob additionally stops the hourly cleanup sweeps and
+            # shared-bucket meta backups. Garbage is reclaimed by an external
+            # reaper that unions live-branch references; never `juicefs gc`.
+            "--max-deletes",
+            "0",
+            "--no-bgjob",
             "--cache-dir",
             config.cache_dir,
             "--max-uploads",

{flyteplugins_union-0.5.0 → flyteplugins_union-0.5.2}/src/flyteplugins/union/io/_internal/_juicefs/_redis.py

@@ -31,7 +31,9 @@ def _free_tcp_port() -> int:
 def _redis_server_cmd(port: int, dirpath: str, dbfilename: str) -> list[str]:
     """Argv for an in-process ``redis-server`` bound to localhost on ``port``,
     persisting ``dbfilename`` under ``dirpath``. RDB-only (no AOF), no periodic
-    save — we drive persistence explicitly via ``SAVE``.
+    save — we drive persistence explicitly via ``BGSAVE`` (see
+    :func:`_bgsave_and_wait`, which relies on ours being the only dump in
+    flight).
     """
     return [
         "redis-server",
@@ -82,7 +84,21 @@ async def _redis_cli(port: int, *args: str, check: bool = False, timeout: float
     return rc, stdout
 
 
-async def _wait_redis_ready(proc: subprocess.Popen, port: int, timeout: float) -> None:
+def _tail_log(log_path: Optional[Path], max_bytes: int = 4096) -> str:
+    """Best-effort read of the last ``max_bytes`` of a daemon log file."""
+    if log_path is None:
+        return ""
+    try:
+        with open(log_path, "rb") as f:
+            f.seek(0, 2)
+            size = f.tell()
+            f.seek(max(0, size - max_bytes))
+            return f.read().decode(errors="replace")
+    except OSError:
+        return ""
+
+
+async def _wait_redis_ready(proc: subprocess.Popen, port: int, timeout: float, log_path: Optional[Path] = None) -> None:
     """Poll ``PING`` until the server replies ``PONG`` (or ``timeout``).
 
     Raises :class:`RuntimeError` if the server exits before becoming ready and
@@ -92,9 +108,9 @@ async def _wait_redis_ready(proc: subprocess.Popen, port: int, timeout: float) -
     deadline = asyncio.get_running_loop().time() + timeout
     while True:
         if proc.poll() is not None:
-            out = proc.stdout.read().decode(errors="replace") if proc.stdout else ""
             raise RuntimeSystemError(
-                "RedisStartFailed", f"redis-server exited prematurely (rc={proc.returncode}): {out}"
+                "RedisStartFailed",
+                f"redis-server exited prematurely (rc={proc.returncode}): {_tail_log(log_path)}",
             )
         rc, stdout = await _redis_cli(port, "ping", timeout=5.0)
         if rc == 0 and "PONG" in stdout:
@@ -125,14 +141,24 @@ class RedisMetadataStore:
     async def start(self, meta_dir: str, timeout: float = 30.0) -> _MetadataStoreHandle:
         port = _free_tcp_port()
         _logger.info("[Volume.mount] starting redis-server in %s", meta_dir)
-        proc = subprocess.Popen(  # noqa: ASYNC220 - long-lived daemon held in a class-level dict
-            _redis_server_cmd(port, meta_dir, _INDEX_FILENAME),
-            stdout=subprocess.PIPE,
-            stderr=subprocess.STDOUT,
-            start_new_session=True,
-        )
-        await _wait_redis_ready(proc, port, timeout)
-        _logger.info("[Volume.mount] redis-server ready on port %d", port)
+        # Route the daemon's logs to a file, never a captured pipe: redis-server
+        # logs for its whole lifetime (background saves etc.), and a PIPE that
+        # nothing drains fills its ~64 KiB kernel buffer — redis then blocks
+        # inside its log write, every metadata op stalls behind it, and the
+        # whole mount wedges. Same failure mode (and same fix) as the mount
+        # daemon's stdio in ``_start_mount``; a file also preserves premature-
+        # exit diagnostics, which inherited stdio would scatter into the
+        # container log.
+        log_path = Path(meta_dir) / "redis-server.log"
+        with open(log_path, "ab") as logf:  # noqa: ASYNC230 - local append-open, microseconds
+            proc = subprocess.Popen(  # noqa: ASYNC220 - long-lived daemon held in a class-level dict
+                _redis_server_cmd(port, meta_dir, _INDEX_FILENAME),
+                stdout=logf,
+                stderr=subprocess.STDOUT,
+                start_new_session=True,
+            )
+        await _wait_redis_ready(proc, port, timeout, log_path)
+        _logger.info("[Volume.mount] redis-server ready on port %d (logs: %s)", port, log_path)
         return _MetadataStoreHandle(proc=proc, port=port)
 
     async def stop(self, handle: Optional[_MetadataStoreHandle], timeout: float) -> None:
@@ -141,7 +167,7 @@ class RedisMetadataStore:
         """
         if handle is None:
             return
-        # SHUTDOWN NOSAVE — we drove persistence ourselves via SAVE. redis-cli
+        # SHUTDOWN NOSAVE — we drove persistence ourselves via BGSAVE. redis-cli
         # may return non-zero as the connection drops mid-shutdown; that's
         # expected, so this call is unchecked. Never let a hung/failed shutdown
         # leave the daemon unreaped — fall through to the wait + kill below.
@@ -157,7 +183,7 @@ class RedisMetadataStore:
     async def flush(self, meta_dir: str, handle: Optional[_MetadataStoreHandle] = None) -> None:
         if handle is None:
             raise RuntimeError("RedisMetadataStore.flush requires a live handle from start().")
-        await _redis_cli(handle.port, "SAVE", check=True)
+        await _bgsave_and_wait(handle.port)
 
     async def snapshot(self, meta_dir: str, tmp_prefix: str) -> str:
         src = Path(meta_dir) / _INDEX_FILENAME
@@ -176,6 +202,49 @@ class RedisMetadataStore:
         return await _disjoint_counters_redis(snapshot_path, offset)
 
 
+async def _bgsave_and_wait(port: int, timeout: float = 600.0, poll_interval: float = 0.2) -> None:
+    """Persist the dataset to the RDB via ``BGSAVE`` and wait for completion.
+
+    Replaces blocking ``SAVE``, which had two failure modes: (a) it stalls the
+    (single-threaded) server for the whole dump, so every metadata op — and
+    therefore every FUSE op on a live mount — hangs during a keep-alive
+    commit; (b) ``redis-cli SAVE`` ran under the default 10s client timeout,
+    so a dump that took longer killed the cli, failed the flush, and aborted
+    the commit spuriously while the server kept SAVEing.
+
+    ``BGSAVE`` forks; the parent keeps serving. Our servers run with
+    ``--save ""`` (no periodic saves), so the only BGSAVE in flight is ours:
+    completion is "``rdb_bgsave_in_progress`` back to 0", and correctness is
+    ``rdb_last_bgsave_status:ok``. If a BGSAVE is already running (commit
+    racing a previous flush), the initial ``BGSAVE`` command reports the
+    schedule error but the poll below still tracks the in-flight dump; we
+    re-issue once it clears rather than fail.
+    """
+    loop = asyncio.get_running_loop()
+    deadline = loop.time() + timeout
+    rc, out = await _redis_cli(port, "BGSAVE")
+    already_running = rc != 0 or "in progress" in out.lower()
+    while True:
+        _, info = await _redis_cli(port, "INFO", "persistence", check=True)
+        fields = dict(
+            line.split(":", 1) for line in info.splitlines() if ":" in line and not line.startswith("#")
+        )
+        if fields.get("rdb_bgsave_in_progress", "").strip() == "0":
+            if already_running:
+                # The dump we watched finish was a racing predecessor's; run
+                # our own so this flush's writes are guaranteed on disk.
+                already_running = False
+                await _redis_cli(port, "BGSAVE", check=True)
+                continue
+            status = fields.get("rdb_last_bgsave_status", "").strip()
+            if status != "ok":
+                raise RuntimeSystemError("RedisBgsaveFailed", f"BGSAVE finished with status {status!r}")
+            return
+        if loop.time() >= deadline:
+            raise RuntimeSystemError("RedisBgsaveTimeout", f"BGSAVE did not complete within {timeout}s")
+        await asyncio.sleep(poll_interval)
+
+
 async def _disjoint_counters_redis(rdb_path: str, offset: int, timeout: float = 30.0) -> int:
     if offset <= 0:
         raise ValueError(f"offset must be positive, got {offset}")
@@ -184,14 +253,17 @@ async def _disjoint_counters_redis(rdb_path: str, offset: int, timeout: float =
         raise RuntimeError(f"{rdb_path} does not exist")
 
     port = _free_tcp_port()
-    proc = subprocess.Popen(  # noqa: ASYNC220 - ephemeral daemon, reaped in finally
-        _redis_server_cmd(port, str(rdb.parent), rdb.name),
-        stdout=subprocess.PIPE,
-        stderr=subprocess.STDOUT,
-        start_new_session=True,
-    )
+    # File-backed logs for the same no-undrained-pipe reason as ``start()``.
+    log_path = rdb.parent / f"{rdb.name}.redis-fork.log"
+    with open(log_path, "ab") as logf:  # noqa: ASYNC230 - local append-open, microseconds
+        proc = subprocess.Popen(  # noqa: ASYNC220 - ephemeral daemon, reaped in finally
+            _redis_server_cmd(port, str(rdb.parent), rdb.name),
+            stdout=logf,
+            stderr=subprocess.STDOUT,
+            start_new_session=True,
+        )
     try:
-        await _wait_redis_ready(proc, port, timeout)
+        await _wait_redis_ready(proc, port, timeout, log_path)
 
         current_max = 0
         for name in _FORK_COUNTER_NAMES:
@@ -208,7 +280,7 @@ async def _disjoint_counters_redis(rdb_path: str, offset: int, timeout: float =
         for name in _FORK_COUNTER_NAMES:
             await _redis_cli(port, "INCRBY", name, str(applied), check=True)
 
-        await _redis_cli(port, "SAVE", check=True)
+        await _bgsave_and_wait(port, timeout=timeout)
     finally:
         # Best-effort shutdown — never let cleanup mask the real error.
         try:
@@ -218,4 +290,8 @@ async def _disjoint_counters_redis(rdb_path: str, offset: int, timeout: float =
         if not await _await_proc_exit(proc, 5):
             proc.kill()
             await _await_proc_exit(proc, 5)
+        try:
+            log_path.unlink()
+        except OSError:
+            pass
     return applied

{flyteplugins_union-0.5.0 → flyteplugins_union-0.5.2}/src/flyteplugins/union/io/_internal/_juicefs/_sqlite.py

@@ -58,6 +58,18 @@ class SqliteMetadataStore:
         # caller runs it off the loop. Kept sync so the sync CLI path can call
         # it directly without an event loop.
         dst = Path(meta_dir) / _INDEX_FILENAME
+        # Scrub sidecars from any previous incarnation of this meta_dir (a
+        # remount after commit, or a task retry reusing the pod). SQLite pairs
+        # a WAL/SHM with its main file via a header salt, but a leftover
+        # ``-wal`` written by an *older* database at this same path can pass
+        # that check and get replayed over the fresh snapshot, silently mixing
+        # two versions' metadata. The snapshot itself is complete (backup API
+        # folds the WAL in), so stale sidecars are never legitimate here.
+        for sidecar in (dst, Path(str(dst) + "-wal"), Path(str(dst) + "-shm")):
+            try:
+                sidecar.unlink()
+            except FileNotFoundError:
+                pass
         shutil.move(snapshot_path, str(dst))
 
     async def disjoint_fork_counters(self, snapshot_path: str, offset: int) -> int:

{flyteplugins_union-0.5.0 → flyteplugins_union-0.5.2}/src/flyteplugins/union/remote/_role.py

@@ -13,11 +13,12 @@ from flyteplugins.union.internal.common.identifier_pb2 import RoleIdentifier
 from flyteplugins.union.internal.common.role_pb2 import ROLE_TYPE_CUSTOM, RoleSpec, RoleType
 from flyteplugins.union.internal.common.role_pb2 import Role as RolePb2
 from flyteplugins.union.internal.identity.role_payload_pb2 import (
+    CreateRoleActionRequest,
     CreateRoleRequest,
+    DeleteRoleActionRequest,
     DeleteRoleRequest,
     GetRoleRequest,
     ListRolesRequest,
-    UpdateRoleRequest,
 )
 from flyteplugins.union.internal.identity.role_service_connect import RoleServiceClient
 
@@ -112,22 +113,29 @@ class Role(ToJSONMixin):
 
     @syncify
     @classmethod
-    async def update(cls, name: str, *, description: str = "", actions: list[str] | None = None) -> None:
-        """Update a role."""
+    async def update(cls, name: str, *, actions: list[str]) -> None:
+        """Update a role so its actions match exactly the given list.
+
+        The UpdateRole RPC is deprecated and rejected server-side, so the
+        current actions are diffed against the desired ones and each delta is
+        applied via CreateRoleAction/DeleteRoleAction. The role name and
+        description cannot be changed.
+        """
         ensure_client()
         client = get_client()
         org = get_init_config().org
         svc = RoleServiceClient(**client.session_config.connect_kwargs())
 
-        resolved = [Action.Value(_resolve_action_name(a)) for a in (actions or [])]
-        role = RolePb2(
-            id=RoleIdentifier(organization=org, name=name),
-            role_spec=RoleSpec(description=description),
-            role_type=ROLE_TYPE_CUSTOM,
-            actions=resolved,
-        )
-        request = UpdateRoleRequest(role=role)
-        await svc.update_role(request)
+        role_id = RoleIdentifier(organization=org, name=name)
+        response = await svc.get_role(GetRoleRequest(id=role_id))
+
+        desired = {Action.Value(_resolve_action_name(a)) for a in actions}
+        current = set(response.role.actions)
+
+        for action in sorted(desired - current):
+            await svc.create_role_action(CreateRoleActionRequest(id=role_id, action=action))
+        for action in sorted(current - desired):
+            await svc.delete_role_action(DeleteRoleActionRequest(id=role_id, action=action))
 
     @syncify
     @classmethod

{flyteplugins_union-0.5.0 → flyteplugins_union-0.5.2/src/flyteplugins_union.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flyteplugins-union
-Version: 0.5.0
+Version: 0.5.2
 Summary: Union SDK - Proprietary extensions for Flyte
 Author-email: unionai <info@union.ai>
 Project-URL: Homepage, https://www.union.ai/

flyteplugins_union-0.5.2/src/flyteplugins_union.egg-info/scm_version.json

@@ -0,0 +1,8 @@
+{
+  "tag": "0.5.2",
+  "distance": 0,
+  "node": "gea5e734894bf3b85e126c462109c380ee9a427e7",
+  "dirty": false,
+  "branch": "HEAD",
+  "node_date": "2026-07-13"
+}

flyteplugins_union-0.5.0/src/flyteplugins_union.egg-info/scm_version.json

@@ -1,8 +0,0 @@
-{
-  "tag": "0.5.0",
-  "distance": 0,
-  "node": "gc211548ea59e46226bb178f2cc517031b94c57c4",
-  "dirty": false,
-  "branch": "HEAD",
-  "node_date": "2026-06-29"
-}