ginkgo-oci 0.4.0__tar.gz

ginkgo_oci-0.4.0/.gitignore

@@ -0,0 +1,58 @@
+# Local environment and tool caches
+.pixi/*
+!.pixi/config.toml
+.pytest_cache/
+.ruff_cache/
+ui/node_modules/
+ui/.vite/
+__pycache__/
+*.py[cod]
+
+*__pycache__*
+# Ginkgo runtime state
+.ginkgo/
+examples/**/logs/
+examples/**/results/
+benchmarks/results/
+
+# Agent / local-instructions files
+AGENTS.md
+# ...but not the one `ginkgo init` scaffolds into new projects.
+!src/ginkgo/templates/init/agents/AGENTS.md
+CLAUDE.md
+.claude/
+.agents/
+
+# uncoded symbol index (regenerated locally via `uncoded sync`)
+.uncoded/
+
+# Local planning / design / worklog docs
+plan.md
+*plan.local.md
+worklog.md
+*worklog*.md
+ginkgo-design-doc.md
+example-workflow.py
+docs/_build/
+docs/site/node_modules/
+docs/site/dist/
+docs/site/.astro/
+
+# OS/editor noise
+.DS_Store
+
+*.lock
+# Env lockfiles that are part of a fixture or example are inputs, not build
+# output: an example whose lock is missing re-solves on first use, which
+# changes its env identity between runs and silently breaks cache reuse.
+!tests/**/pixi.lock
+!plugins/**/pixi.lock
+!examples/**/pixi.lock
+
+# Maintainer working documents (CLAUDE.md: plans are not committed)
+docs/*-plan.md
+docs/user-testing-*.md
+docs/oke-validation.md
+docs/*.pptx
+scratchpad/
+.vscode/

ginkgo_oci-0.4.0/PKG-INFO

@@ -0,0 +1,15 @@
+Metadata-Version: 2.5
+Name: ginkgo-oci
+Version: 0.4.0
+Summary: Oracle Cloud integrations for ginkgo: the oci:// object store and rclone driver
+License-Expression: Apache-2.0
+Requires-Python: >=3.11
+Requires-Dist: ginkgo-workflow==0.4.0
+Requires-Dist: ocifs>=1.3.2
+Description-Content-Type: text/markdown
+
+# ginkgo-oci
+
+Oracle Cloud integrations for ginkgo: the oci:// object store and rclone driver.
+
+A plugin for [Ginkgo](https://sanjaynagi.github.io/ginkgo/). Install it with `pip install "ginkgo-workflow[oci]"`, and see the [plugins guide](https://sanjaynagi.github.io/ginkgo/getting-started/installation/#plugins).

ginkgo_oci-0.4.0/pyproject.toml

@@ -0,0 +1,21 @@
+[project]
+name = "ginkgo-oci"
+version = "0.4.0"
+description = "Oracle Cloud integrations for ginkgo: the oci:// object store and rclone driver"
+requires-python = ">= 3.11"
+readme = { text = "# ginkgo-oci\n\nOracle Cloud integrations for ginkgo: the oci:// object store and rclone driver.\n\nA plugin for [Ginkgo](https://sanjaynagi.github.io/ginkgo/). Install it with `pip install \"ginkgo-workflow[oci]\"`, and see the [plugins guide](https://sanjaynagi.github.io/ginkgo/getting-started/installation/#plugins).", content-type = "text/markdown" }
+license = "Apache-2.0"
+dependencies = [
+    "ginkgo-workflow==0.4.0",
+    "ocifs>=1.3.2",
+]
+
+[project.entry-points."ginkgo.plugins"]
+oci = "ginkgo_oci:PLUGIN"
+
+[build-system]
+build-backend = "hatchling.build"
+requires = ["hatchling"]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/ginkgo_oci"]

ginkgo_oci-0.4.0/src/ginkgo_oci/__init__.py

@@ -0,0 +1,17 @@
+"""ginkgo-oci: the ``oci://`` object store and the rclone FUSE driver."""
+
+from __future__ import annotations
+
+from ginkgo.plugins.api import MountDriverType, ObjectStoreType, Plugin
+from ginkgo_oci.rclone import RcloneDriver
+from ginkgo_oci.store import build_oci_store, filesystem, parse_oci_uri
+
+__all__ = ["PLUGIN", "filesystem"]
+
+PLUGIN = Plugin(
+    name="ginkgo-oci",
+    contributions=(
+        ObjectStoreType(scheme="oci", parse_uri=parse_oci_uri, build=build_oci_store),
+        MountDriverType(scheme="oci", build=RcloneDriver),
+    ),
+)

ginkgo_oci-0.4.0/src/ginkgo_oci/rclone.py

@@ -0,0 +1,106 @@
+"""``rclone mount`` driver wrapper, ginkgo-oci's driver for ``oci://`` mounts.
+
+rclone's per-remote configuration is the user's responsibility; we assume a
+remote named ``<scheme>`` is configured.
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+import subprocess
+from pathlib import Path
+
+from ginkgo.remote.access.drivers.base import (
+    DriverUnavailableError,
+    MountFailedError,
+    MountSpec,
+    unmount_fuse,
+)
+
+
+class RcloneDriver:
+    """Thin subprocess wrapper around ``rclone mount``."""
+
+    name = "rclone"
+
+    def __init__(self, *, binary: str = "rclone", remote: str | None = None) -> None:
+        self._binary = binary
+        self._remote = remote
+
+    def health_check(self) -> None:
+        """Verify ``rclone`` is on PATH."""
+        if shutil.which(self._binary) is None:
+            raise DriverUnavailableError(
+                f"{self._binary!r} binary not found on PATH. "
+                "Install rclone or use the worker-fuse image."
+            )
+
+    def mount(self, *, spec: MountSpec) -> int:
+        """Mount ``<remote>:<bucket>`` at ``spec.mount_point``."""
+        spec.mount_point.mkdir(parents=True, exist_ok=True)
+        remote_name = self._remote or spec.scheme
+
+        # OCI refs carry the namespace in the bucket component
+        # ("bucket@namespace"), but rclone's oracleobjectstorage backend
+        # takes the namespace from the remote's config and rejects "@" in
+        # bucket names. Mount only the bucket component; the configured
+        # remote must point at the matching namespace.
+        bucket = spec.bucket.split("@", 1)[0]
+
+        cmd = [
+            self._binary,
+            "mount",
+            f"{remote_name}:{bucket}",
+            str(spec.mount_point),
+            "--daemon",
+        ]
+        if spec.read_only:
+            cmd.append("--read-only")
+        if spec.cache_dir is not None:
+            spec.cache_dir.mkdir(parents=True, exist_ok=True)
+            cmd += [
+                "--cache-dir",
+                str(spec.cache_dir),
+                "--vfs-cache-mode",
+                "full",
+            ]
+            if spec.cache_max_bytes is not None:
+                cmd += [
+                    "--vfs-cache-max-size",
+                    f"{max(1, spec.cache_max_bytes // (1024 * 1024))}M",
+                ]
+        cmd += list(spec.extra_args)
+
+        try:
+            result = subprocess.run(
+                cmd,
+                check=False,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+            )
+        except FileNotFoundError as exc:
+            raise DriverUnavailableError(str(exc)) from exc
+
+        if result.returncode != 0:
+            raise MountFailedError(
+                "rclone mount failed: "
+                f"rc={result.returncode} stderr={result.stderr.decode(errors='replace')}"
+            )
+
+        # The daemon exits 0 even when the remote is misconfigured (bad
+        # bucket, bad credentials) and reads then fail with EIO. Probe the
+        # mount with one directory listing so a broken mount surfaces as
+        # MountFailedError, letting hydration fall back to staging.
+        try:
+            os.listdir(spec.mount_point)
+        except OSError as exc:
+            self.unmount(mount_point=spec.mount_point)
+            raise MountFailedError(
+                f"rclone mount at {spec.mount_point} is not readable: {exc}"
+            ) from exc
+        return 0
+
+    def unmount(self, *, mount_point: Path) -> None:
+        """Unmount via the generic helper."""
+        unmount_fuse(mount_point=mount_point)

ginkgo_oci-0.4.0/src/ginkgo_oci/store.py

@@ -0,0 +1,166 @@
+"""The ``oci://`` object store, on ``ocifs``, and its URI layout."""
+
+from __future__ import annotations
+
+import os
+from typing import Any
+from urllib.parse import urlparse
+
+from ginkgo.core.remote import RemoteLocation
+from ginkgo.plugins.api import ObjectStoreContext
+from ginkgo.remote.backend import ObjectStore
+from ginkgo.remote.fsspec_backends import FsspecBackend
+from ginkgo.remote.resolve import config_string, config_table, first_defined, resolve_backend
+
+# How a store signs its requests, by the name ginkgo's config uses, with the
+# ``iam_type`` ocifs takes for it.
+AUTH_IAM_TYPES = {
+    "api_key": "api_key",  # pragma: allowlist secret
+    "oke_workload_identity": "oke_principal",
+    "instance_principal": "instance_principal",
+    "resource_principal": "resource_principal",
+}
+
+
+class OCIFileSystemBackend(FsspecBackend):
+    """OCI Object Storage backend implemented with ``ocifs``.
+
+    Parameters
+    ----------
+    auth : str
+        How requests are signed: a key of :data:`AUTH_IAM_TYPES`. Only
+        ``api_key`` reads a config file.
+    config_path : str | None
+        Path to the OCI config file, for ``api_key`` auth.
+    profile : str | None
+        OCI config profile name, for ``api_key`` auth.
+    region : str | None
+        OCI region override.
+    """
+
+    def __init__(
+        self,
+        *,
+        auth: str = "api_key",
+        config_path: str | None = None,
+        profile: str | None = None,
+        region: str | None = None,
+    ) -> None:
+        super().__init__(scheme="oci")
+        if auth not in AUTH_IAM_TYPES:
+            raise ValueError(
+                f"Unknown OCI auth {auth!r}; expected one of {', '.join(sorted(AUTH_IAM_TYPES))}"
+            )
+        self._auth = auth
+        self._config_path = config_path
+        self._profile = profile
+        self._region = region
+
+    def filesystem(self) -> Any:
+        """Return the ``ocifs.OCIFileSystem`` this store reads and writes through."""
+        return self._get_filesystem()
+
+    def _get_filesystem(self) -> Any:
+        if self._filesystem is not None:
+            return self._filesystem
+
+        # Imported on first use: the plugin module loads on every CLI run.
+        import ocifs
+
+        kwargs: dict[str, Any] = {"iam_type": AUTH_IAM_TYPES[self._auth]}
+        if self._auth == "api_key":
+            if self._config_path is not None:
+                kwargs["config"] = os.path.expanduser(self._config_path)
+            if self._profile is not None:
+                kwargs["profile"] = self._profile
+        if self._region is not None:
+            kwargs["region"] = self._region
+
+        self._filesystem = ocifs.OCIFileSystem(**kwargs)
+        return self._filesystem
+
+
+def filesystem() -> Any:
+    """Return an ``ocifs.OCIFileSystem`` signed the way the ``oci://`` store is.
+
+    For workflow code that reads or writes OCI Object Storage itself, such as a
+    task uploading its results to a staging prefix, so it uses the auth, profile
+    and region the run is configured with.
+
+    Returns
+    -------
+    ocifs.OCIFileSystem
+        The configured filesystem.
+    """
+    backend = resolve_backend("oci")
+    assert isinstance(backend, OCIFileSystemBackend)
+    return backend.filesystem()
+
+
+def build_oci_store(context: ObjectStoreContext) -> ObjectStore:
+    """Build the OCI store from ``GINKGO_REMOTE_OCI_*``/``OCI_*`` or ``[remote.oci]``."""
+    remote = config_table(context.config, "remote")
+    oci = config_table(remote, "oci")
+    return OCIFileSystemBackend(
+        auth=first_defined(
+            os.environ.get("GINKGO_REMOTE_OCI_AUTH"),
+            config_string(oci, "auth"),
+        )
+        or "api_key",
+        config_path=first_defined(
+            os.environ.get("GINKGO_REMOTE_OCI_CONFIG"),
+            os.environ.get("OCI_CONFIG_FILE"),
+            config_string(oci, "config"),
+            "~/.oci/config",
+        ),
+        profile=first_defined(
+            os.environ.get("GINKGO_REMOTE_OCI_PROFILE"),
+            os.environ.get("OCI_CONFIG_PROFILE"),
+            config_string(oci, "profile"),
+        ),
+        region=context.region
+        or first_defined(
+            os.environ.get("GINKGO_REMOTE_OCI_REGION"),
+            os.environ.get("OCI_REGION"),
+            config_string(oci, "region"),
+            config_string(remote, "region"),
+        ),
+    )
+
+
+def parse_oci_uri(uri: str) -> RemoteLocation:
+    """Parse an OCI Object Storage URI.
+
+    Both forms are accepted: ``oci://namespace/bucket/key`` and
+    ``oci://bucket@namespace/key``. The bucket is normalised to
+    ``"<bucket>@<namespace>"``, the authority ``ocifs`` addresses.
+
+    Raises
+    ------
+    ValueError
+        If the namespace, bucket or key is missing.
+    """
+    parsed = urlparse(uri)
+    authority = parsed.netloc
+    if not authority:
+        raise ValueError(f"OCI URI missing namespace or bucket: {uri!r}")
+
+    form = "oci://namespace/bucket/key or oci://bucket@namespace/key"
+    key = parsed.path.lstrip("/")
+    if not key:
+        raise ValueError(f"OCI URI must have the form {form}: {uri!r}")
+
+    if "@" in authority:
+        bucket, namespace = authority.split("@", 1)
+        if not bucket or not namespace:
+            raise ValueError(f"OCI URI must have the form oci://bucket@namespace/key: {uri!r}")
+        return RemoteLocation(
+            scheme="oci", bucket=f"{bucket}@{namespace}", key=key, namespace=namespace
+        )
+
+    bucket, _, object_key = key.partition("/")
+    if not object_key:
+        raise ValueError(f"OCI URI must have the form {form}: {uri!r}")
+    return RemoteLocation(
+        scheme="oci", bucket=f"{bucket}@{authority}", key=object_key, namespace=authority
+    )

ginkgo_oci-0.4.0/tests/test_oci_store.py

@@ -0,0 +1,117 @@
+"""The oci:// object store contributed by ginkgo-oci, and its URI layout."""
+
+from __future__ import annotations
+
+import pytest
+
+from ginkgo.core.remote import RemoteFileRef, RemoteFolderRef, remote_file, remote_folder
+from ginkgo.remote.fsspec_backends import FsspecBackend
+from ginkgo.remote.resolve import resolve_backend
+from ginkgo_oci.store import OCIFileSystemBackend
+
+
+class TestOCIUris:
+    def test_oci_uri(self) -> None:
+        ref = remote_file("oci://mynamespace/mybucket/path/to/object.csv")
+        assert isinstance(ref, RemoteFileRef)
+        assert ref.scheme == "oci"
+        assert ref.bucket == "mybucket@mynamespace"
+        assert ref.namespace == "mynamespace"
+        assert ref.key == "path/to/object.csv"
+
+    def test_oci_uri_nested_key(self) -> None:
+        ref = remote_file("oci://ns/bkt/a/b/c.txt")
+        assert ref.bucket == "bkt@ns"
+        assert ref.namespace == "ns"
+        assert ref.key == "a/b/c.txt"
+
+    def test_oci_bucket_at_namespace_uri(self) -> None:
+        ref = remote_file("oci://mybucket@mynamespace/path/to/object.csv")
+        assert ref.bucket == "mybucket@mynamespace"
+        assert ref.namespace == "mynamespace"
+        assert ref.key == "path/to/object.csv"
+
+    def test_oci_missing_namespace_raises(self) -> None:
+        with pytest.raises(ValueError, match="missing namespace"):
+            remote_file("oci:///bucket/key")
+
+    def test_oci_missing_key_raises(self) -> None:
+        with pytest.raises(ValueError, match="oci://namespace/bucket/key"):
+            remote_file("oci://namespace/bucket")
+
+    def test_oci_prefix(self) -> None:
+        ref = remote_folder("oci://ns/bkt/prefix/")
+        assert isinstance(ref, RemoteFolderRef)
+        assert ref.bucket == "bkt@ns"
+        assert ref.key == "prefix/"
+
+
+class TestBuildOCIStore:
+    def test_profile_comes_from_the_environment(self, monkeypatch: pytest.MonkeyPatch) -> None:
+        monkeypatch.setenv("GINKGO_REMOTE_OCI_PROFILE", "uk")
+
+        backend = resolve_backend("oci")
+
+        assert isinstance(backend, OCIFileSystemBackend)
+        assert isinstance(backend, FsspecBackend)
+        assert backend._scheme == "oci"
+        assert backend._profile == "uk"
+
+
+class TestOCIAuth:
+    def _built_kwargs(self, monkeypatch: pytest.MonkeyPatch) -> dict:
+        """Return the keyword arguments the store passes to ``OCIFileSystem``."""
+        import ocifs
+
+        captured: dict = {}
+        monkeypatch.setattr(ocifs, "OCIFileSystem", lambda **kwargs: captured.update(kwargs))
+        resolve_backend("oci").filesystem()
+        return captured
+
+    def test_workload_identity_signs_without_a_config_file(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        monkeypatch.setenv("GINKGO_REMOTE_OCI_AUTH", "oke_workload_identity")
+        monkeypatch.setenv("GINKGO_REMOTE_OCI_REGION", "uk-london-1")
+        monkeypatch.setenv("GINKGO_REMOTE_OCI_PROFILE", "ignored")
+
+        assert self._built_kwargs(monkeypatch) == {
+            "iam_type": "oke_principal",
+            "region": "uk-london-1",
+        }
+
+    def test_api_key_is_the_default_and_reads_the_config_file(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        monkeypatch.setenv("GINKGO_REMOTE_OCI_CONFIG", "/etc/oci/config")
+        monkeypatch.setenv("GINKGO_REMOTE_OCI_PROFILE", "uk")
+
+        assert self._built_kwargs(monkeypatch) == {
+            "iam_type": "api_key",
+            "config": "/etc/oci/config",
+            "profile": "uk",
+        }
+
+    def test_auth_can_come_from_config(self, tmp_path, monkeypatch: pytest.MonkeyPatch) -> None:
+        (tmp_path / "ginkgo.toml").write_text(
+            '[remote.oci]\nauth = "instance_principal"\n', encoding="utf-8"
+        )
+        monkeypatch.chdir(tmp_path)
+
+        assert self._built_kwargs(monkeypatch)["iam_type"] == "instance_principal"
+
+    def test_an_unknown_auth_is_refused(self, monkeypatch: pytest.MonkeyPatch) -> None:
+        monkeypatch.setenv("GINKGO_REMOTE_OCI_AUTH", "password")
+
+        with pytest.raises(ValueError, match="Unknown OCI auth 'password'"):
+            resolve_backend("oci")
+
+    def test_filesystem_returns_the_configured_one(self, monkeypatch: pytest.MonkeyPatch) -> None:
+        import ocifs
+
+        from ginkgo_oci import filesystem
+
+        sentinel = object()
+        monkeypatch.setattr(ocifs, "OCIFileSystem", lambda **kwargs: sentinel)
+
+        assert filesystem() is sentinel

ginkgo_oci-0.4.0/tests/test_rclone.py

@@ -0,0 +1,61 @@
+"""The rclone FUSE driver ginkgo-oci uses for oci:// mounts."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+import ginkgo_oci.rclone as rclone_mod
+from ginkgo.remote.access.drivers.base import MountFailedError, MountSpec
+
+
+class TestRcloneDriver:
+    def _spec(self, tmp_path: Path, *, bucket: str) -> MountSpec:
+        return MountSpec(
+            scheme="oci",
+            bucket=bucket,
+            mount_point=tmp_path / "mount",
+            cache_dir=None,
+            cache_max_bytes=None,
+            read_only=True,
+        )
+
+    def test_mount_strips_oci_namespace_from_bucket(self, tmp_path: Path, monkeypatch) -> None:
+        commands: list[list[str]] = []
+
+        def _fake_run(cmd, **kwargs):  # noqa: ARG001
+            commands.append(cmd)
+
+            class _Result:
+                returncode = 0
+                stderr = b""
+
+            return _Result()
+
+        monkeypatch.setattr(rclone_mod.subprocess, "run", _fake_run)
+        driver = rclone_mod.RcloneDriver()
+        driver.mount(spec=self._spec(tmp_path, bucket="my-bucket@my-namespace"))
+        assert commands[0][2] == "oci:my-bucket"
+
+    def test_mount_probe_failure_raises_and_unmounts(self, tmp_path: Path, monkeypatch) -> None:
+        def _fake_run(cmd, **kwargs):  # noqa: ARG001
+            class _Result:
+                returncode = 0
+                stderr = b""
+
+            return _Result()
+
+        def _fail_listdir(path):  # noqa: ARG001
+            raise OSError(5, "Input/output error")
+
+        unmounted: list[Path] = []
+        monkeypatch.setattr(rclone_mod.subprocess, "run", _fake_run)
+        monkeypatch.setattr(rclone_mod.os, "listdir", _fail_listdir)
+        driver = rclone_mod.RcloneDriver()
+        monkeypatch.setattr(
+            driver, "unmount", lambda *, mount_point: unmounted.append(mount_point)
+        )
+        with pytest.raises(MountFailedError, match="not readable"):
+            driver.mount(spec=self._spec(tmp_path, bucket="my-bucket@my-namespace"))
+        assert unmounted == [tmp_path / "mount"]