ondio 0.1.0__py3-none-any.whl

ondio/__init__.py

@@ -0,0 +1,53 @@
+"""ondio: uniform IO for audio data across AWS S3, GCS, HTTP, and local files.
+
+One URI-addressed interface; the platform is detected from the scheme and the
+call is dispatched to the matching backend. See PROJECT.md for the full design.
+"""
+
+from ondio.dispatcher import (
+    delete,
+    download,
+    download_flac,
+    download_json,
+    exists,
+    extract_flac_header,
+    list_files,
+    object_count,
+    read,
+    read_flac,
+    upload,
+    write,
+    write_json,
+)
+from ondio.registry import detect_platform
+from ondio.types import (
+    AuthError,
+    FlacHeader,
+    ObjectNotFoundError,
+    OndioError,
+    UnknownPlatformError,
+    UnsupportedOperationError,
+)
+
+__all__ = [
+    "AuthError",
+    "FlacHeader",
+    "ObjectNotFoundError",
+    "OndioError",
+    "UnknownPlatformError",
+    "UnsupportedOperationError",
+    "delete",
+    "detect_platform",
+    "download",
+    "download_flac",
+    "download_json",
+    "exists",
+    "extract_flac_header",
+    "list_files",
+    "object_count",
+    "read",
+    "read_flac",
+    "upload",
+    "write",
+    "write_json",
+]

ondio/_log.py

@@ -0,0 +1,46 @@
+"""Lazy, optional logging through cocina's Printer.
+
+cocina's `Printer` is a process-wide singleton: whoever constructs it first
+fixes its configuration (log file, silence, ...), and later constructor
+arguments are silently ignored. This module therefore never constructs it at
+import time — the instance is resolved on each attribute access, so an
+application that configures `Printer(...)` before ondio's first log line
+wins. If cocina is not installed, every call is a silent no-op: logging
+narrates, exceptions are the failure signal, so nothing is lost.
+
+Usage:
+    from ondio._log import printer
+
+    printer.message("using byte-range strategy", start_byte=0, end_byte=1024)
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+try:
+    from cocina.printer import Printer
+except ImportError:
+    Printer = None
+
+
+def _noop(*args: Any, **kwargs: Any) -> None:
+    return None
+
+
+class _LazyPrinter:
+    """Forwards attribute access to the Printer singleton (or a no-op).
+
+    Attribute access returns the singleton's *bound method*, so the call
+    itself puts no ondio._log frame on the stack — cocina's automatic
+    caller-name headers (`caller_name` skips only cocina frames) keep
+    labeling messages with the real calling module.
+    """
+
+    def __getattr__(self, name: str) -> Any:
+        if Printer is None:
+            return _noop
+        return getattr(Printer(), name)
+
+
+printer = _LazyPrinter()

ondio/backends/aws.py

@@ -0,0 +1,161 @@
+"""AWS S3 backend."""
+
+from __future__ import annotations
+
+import contextlib
+import os
+from pathlib import Path
+from urllib.parse import urlparse
+
+from ondio.backends.protocol import _matches_filters
+from ondio.types import AuthError, ObjectNotFoundError, OndioError
+
+_NOT_FOUND_CODES = {"404", "NoSuchKey", "NoSuchBucket"}
+_AUTH_CODES = {"403", "AccessDenied", "InvalidAccessKeyId", "SignatureDoesNotMatch", "ExpiredToken"}
+
+
+def _split(uri: str) -> tuple[str, str]:
+    parsed = urlparse(uri)
+    if parsed.scheme != "s3" or not parsed.netloc:
+        raise ValueError(f"not an s3:// URI: {uri!r}")
+    return parsed.netloc, parsed.path.lstrip("/")
+
+
+class AwsBackend:
+    """StorageBackend for AWS S3 (`s3://bucket/key` URIs), via boto3.
+
+    See [`StorageBackend`][ondio.backends.StorageBackend] for method
+    semantics. Credentials resolve through boto3's native chain; botocore
+    errors are translated into ondio's exception types.
+
+    Args:
+        **session_kwargs: Forwarded to `boto3.Session` (e.g. `profile_name`,
+            `region_name`).
+    """
+
+    def __init__(self, **session_kwargs):
+        import boto3
+
+        self._client = boto3.Session(**session_kwargs).client("s3")
+
+    @contextlib.contextmanager
+    def _translate(self, uri: str):
+        from botocore.exceptions import ClientError
+
+        try:
+            yield
+        except ClientError as exc:
+            code = exc.response.get("Error", {}).get("Code", "")
+            if code in _NOT_FOUND_CODES:
+                raise ObjectNotFoundError(f"no such object: {uri}") from exc
+            if code in _AUTH_CODES:
+                raise AuthError(f"access denied ({code}): {uri}") from exc
+            raise OndioError(f"S3 error ({code}) for {uri}") from exc
+
+    def read(self, uri: str) -> bytes:
+        bucket, key = _split(uri)
+        with self._translate(uri):
+            return self._client.get_object(Bucket=bucket, Key=key)["Body"].read()
+
+    def read_range(self, uri: str, start_byte: int, end_byte: int | None) -> bytes:
+        if start_byte < 0:
+            raise ValueError(f"start_byte must be >= 0, got {start_byte}")
+        if end_byte is not None and end_byte < start_byte:
+            raise ValueError(f"end_byte {end_byte} < start_byte {start_byte}")
+        bucket, key = _split(uri)
+        header = f"bytes={start_byte}-" if end_byte is None else f"bytes={start_byte}-{end_byte}"
+        from botocore.exceptions import ClientError
+
+        try:
+            return self._client.get_object(Bucket=bucket, Key=key, Range=header)["Body"].read()
+        except ClientError as exc:
+            if exc.response.get("Error", {}).get("Code") == "InvalidRange":
+                return b""  # start past EOF: protocol says return the available bytes
+            with self._translate(uri):
+                raise
+
+    def size(self, uri: str) -> int:
+        bucket, key = _split(uri)
+        with self._translate(uri):
+            return self._client.head_object(Bucket=bucket, Key=key)["ContentLength"]
+
+    def write(self, uri: str, data: bytes) -> None:
+        bucket, key = _split(uri)
+        with self._translate(uri):
+            self._client.put_object(Bucket=bucket, Key=key, Body=data)
+
+    def download(self, uri: str, out_path: str | os.PathLike[str]) -> None:
+        bucket, key = _split(uri)
+        dest = Path(out_path)
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        with self._translate(uri):
+            self._client.download_file(bucket, key, str(dest))
+
+    def list_files(
+        self,
+        uri_prefix: str,
+        *,
+        recursive: bool = True,
+        max_items: int | None = None,
+        required_prefix: str | None = None,
+        required_ext: str | None = None,
+    ) -> list[str]:
+        bucket, key = _split(uri_prefix)
+        # required_prefix filters the *filename* of every (recursively listed)
+        # key, so it cannot be folded into the native Prefix — that would miss
+        # nested keys like sub/chunk_003.flac.
+        # Likewise the native MaxItems counts unfiltered keys, so max_items 
+        # must be applied after filtering.
+        filtered = required_prefix is not None or required_ext is not None
+        paginate_kwargs: dict = {"Bucket": bucket, "Prefix": key}
+        if not recursive:
+            paginate_kwargs["Delimiter"] = "/"
+        if max_items is not None and not filtered:
+            paginate_kwargs["PaginationConfig"] = {"MaxItems": max_items}
+
+        results = []
+        with self._translate(uri_prefix):
+            for page in self._client.get_paginator("list_objects_v2").paginate(**paginate_kwargs):
+                for obj in page.get("Contents", []):
+                    if obj["Key"].endswith("/"):  # skip folder placeholders
+                        continue
+                    name = obj["Key"].rsplit("/", 1)[-1]
+                    if _matches_filters(name, required_prefix, required_ext):
+                        results.append(f"s3://{bucket}/{obj['Key']}")
+        results.sort()
+        return results if max_items is None else results[:max_items]
+
+    def object_count(
+        self,
+        uri_prefix: str,
+        *,
+        required_prefix: str | None = None,
+        required_ext: str | None = None,
+    ) -> int:
+        bucket, key = _split(uri_prefix)
+        # required_prefix filters the *filename* of every (recursively listed)
+        # key, so it cannot be folded into the native Prefix — that would miss
+        # nested keys like sub/chunk_003.flac.
+        count = 0
+        with self._translate(uri_prefix):
+            for page in self._client.get_paginator("list_objects_v2").paginate(Bucket=bucket, Prefix=key):
+                if required_prefix is None and required_ext is None:
+                    count += page.get("KeyCount", len(page.get("Contents", [])))
+                    continue
+                for obj in page.get("Contents", []):
+                    name = obj["Key"].rsplit("/", 1)[-1]
+                    if _matches_filters(name, required_prefix, required_ext):
+                        count += 1
+        return count
+
+    def exists(self, uri: str) -> bool:
+        try:
+            self.size(uri)
+            return True
+        except ObjectNotFoundError:
+            return False
+
+    def delete(self, uri: str) -> None:
+        bucket, key = _split(uri)
+        with self._translate(uri):
+            self._client.delete_object(Bucket=bucket, Key=key)  # S3 delete is idempotent

ondio/backends/gcs.py

@@ -0,0 +1,159 @@
+"""Google Cloud Storage backend."""
+
+from __future__ import annotations
+
+import contextlib
+import os
+from pathlib import Path
+from urllib.parse import urlparse
+
+from ondio.backends.protocol import _matches_filters
+from ondio.types import AuthError, ObjectNotFoundError, OndioError
+
+
+def _split(uri: str) -> tuple[str, str]:
+    parsed = urlparse(uri)
+    if parsed.scheme != "gs" or not parsed.netloc:
+        raise ValueError(f"not a gs:// URI: {uri!r}")
+    return parsed.netloc, parsed.path.lstrip("/")
+
+
+class GcsBackend:
+    """StorageBackend for Google Cloud Storage (`gs://bucket/key` URIs).
+
+    See [`StorageBackend`][ondio.backends.StorageBackend] for method
+    semantics. Requires the `gcs` extra. Credentials resolve through the
+    google-cloud-storage client's native chain; API errors are translated
+    into ondio's exception types.
+
+    Args:
+        **client_kwargs: Forwarded to `google.cloud.storage.Client`
+            (e.g. `project`).
+    """
+
+    def __init__(self, **client_kwargs):
+        try:
+            from google.cloud import storage
+        except ImportError as exc:
+            raise ImportError(
+                "gs:// support requires the 'gcs' extra: pip install ondio[gcs]"
+            ) from exc
+        self._client = storage.Client(**client_kwargs)
+
+    @contextlib.contextmanager
+    def _translate(self, uri: str):
+        from google.api_core import exceptions as gexc
+
+        try:
+            yield
+        except gexc.NotFound as exc:
+            raise ObjectNotFoundError(f"no such object: {uri}") from exc
+        except (gexc.Forbidden, gexc.Unauthorized) as exc:
+            raise AuthError(f"access denied: {uri}") from exc
+        except gexc.GoogleAPICallError as exc:
+            raise OndioError(f"GCS error for {uri}: {exc}") from exc
+
+    def _blob(self, uri: str):
+        bucket, key = _split(uri)
+        return self._client.bucket(bucket).blob(key)
+
+    def read(self, uri: str) -> bytes:
+        with self._translate(uri):
+            return self._blob(uri).download_as_bytes()
+
+    def read_range(self, uri: str, start_byte: int, end_byte: int | None) -> bytes:
+        if start_byte < 0:
+            raise ValueError(f"start_byte must be >= 0, got {start_byte}")
+        if end_byte is not None and end_byte < start_byte:
+            raise ValueError(f"end_byte {end_byte} < start_byte {start_byte}")
+        from google.api_core import exceptions as gexc
+
+        with self._translate(uri):
+            try:
+                # google-cloud-storage end is inclusive, matching the protocol
+                return self._blob(uri).download_as_bytes(start=start_byte, end=end_byte)
+            except gexc.RequestRangeNotSatisfiable:
+                # must be caught before _translate sees it: it subclasses
+                # GoogleAPICallError and would be swallowed into OndioError
+                return b""  # start past EOF: protocol says return the available bytes
+
+    def size(self, uri: str) -> int:
+        bucket, key = _split(uri)
+        with self._translate(uri):
+            blob = self._client.bucket(bucket).get_blob(key)
+        if blob is None:
+            raise ObjectNotFoundError(f"no such object: {uri}")
+        return blob.size
+
+    def write(self, uri: str, data: bytes) -> None:
+        with self._translate(uri):
+            self._blob(uri).upload_from_string(data, content_type="application/octet-stream")
+
+    def download(self, uri: str, out_path: str | os.PathLike[str]) -> None:
+        dest = Path(out_path)
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        with self._translate(uri):
+            self._blob(uri).download_to_filename(str(dest))
+
+    def list_files(
+        self,
+        uri_prefix: str,
+        *,
+        recursive: bool = True,
+        max_items: int | None = None,
+        required_prefix: str | None = None,
+        required_ext: str | None = None,
+    ) -> list[str]:
+        bucket, key = _split(uri_prefix)
+        # The filters apply to the *filename* of every (recursively listed)
+        # key, so they cannot be folded into the native prefix, and the native
+        # max_results counts unfiltered keys — cap after filtering instead.
+        filtered = required_prefix is not None or required_ext is not None
+        with self._translate(uri_prefix):
+            blobs = self._client.list_blobs(
+                bucket,
+                prefix=key,
+                delimiter=None if recursive else "/",
+                max_results=None if filtered else max_items,
+            )
+            results = [
+                f"gs://{bucket}/{blob.name}"
+                for blob in blobs
+                if not blob.name.endswith("/")
+                and _matches_filters(blob.name.rsplit("/", 1)[-1], required_prefix, required_ext)
+            ]
+        results.sort()
+        return results if max_items is None else results[:max_items]
+
+    def object_count(
+        self,
+        uri_prefix: str,
+        *,
+        required_prefix: str | None = None,
+        required_ext: str | None = None,
+    ) -> int:
+        bucket, key = _split(uri_prefix)
+        # required_prefix filters the *filename* of every (recursively listed)
+        # key, so it cannot be folded into the native prefix — that would miss
+        # nested keys like sub/chunk_003.flac.
+        count = 0
+        with self._translate(uri_prefix):
+            for blob in self._client.list_blobs(bucket, prefix=key):
+                if _matches_filters(blob.name.rsplit("/", 1)[-1], required_prefix, required_ext):
+                    count += 1
+        return count
+
+    def exists(self, uri: str) -> bool:
+        with self._translate(uri):
+            return self._blob(uri).exists()
+
+    def delete(self, uri: str) -> None:
+        from google.api_core import exceptions as gexc
+
+        try:
+            with self._translate(uri):
+                self._blob(uri).delete()
+        except ObjectNotFoundError:
+            pass  # idempotent delete, per the protocol
+        except gexc.NotFound:  # pragma: no cover - mapped above
+            pass

ondio/backends/http.py

@@ -0,0 +1,127 @@
+"""HTTP/HTTPS backend (read-only).
+
+Write and listing operations are infeasible over plain HTTP and raise
+UnsupportedOperationError. read_range verifies the server honored the Range
+header (206) and refuses to pass off a full-body 200 as a range.
+"""
+
+from __future__ import annotations
+
+import os
+import threading
+from pathlib import Path
+
+from ondio.types import AuthError, ObjectNotFoundError, OndioError, UnsupportedOperationError
+
+_CHUNK = 1024 * 1024
+
+
+class HttpBackend:
+    """Read-only StorageBackend for plain `http(s)://` URLs, via requests.
+
+    See [`StorageBackend`][ondio.backends.StorageBackend] for method
+    semantics. Requires the `http` extra. `upload`, `list_files`,
+    `object_count`, and `delete` raise `UnsupportedOperationError` —
+    infeasible over plain HTTP. `read_range` verifies the server honored the
+    `Range` header (206) and refuses to pass off a full-body 200 as a range.
+    """
+
+    def __init__(self):
+        try:
+            import requests
+        except ImportError as exc:
+            raise ImportError(
+                "http(s):// support requires the 'http' extra: pip install ondio[http]"
+            ) from exc
+        self._requests = requests
+        self._local = threading.local()  # requests.Session is not thread-safe
+
+    @property
+    def _session(self):
+        session = getattr(self._local, "session", None)
+        if session is None:
+            session = self._local.session = self._requests.Session()
+        return session
+
+    def _check_status(self, uri: str, response) -> None:
+        if response.status_code == 404:
+            raise ObjectNotFoundError(f"no such object: {uri}")
+        if response.status_code in (401, 403):
+            raise AuthError(f"access denied ({response.status_code}): {uri}")
+        if response.status_code >= 400:
+            raise OndioError(f"HTTP {response.status_code} for {uri}")
+
+    def read(self, uri: str) -> bytes:
+        response = self._session.get(uri)
+        self._check_status(uri, response)
+        return response.content
+
+    def read_range(self, uri: str, start_byte: int, end_byte: int | None) -> bytes:
+        if start_byte < 0:
+            raise ValueError(f"start_byte must be >= 0, got {start_byte}")
+        if end_byte is not None and end_byte < start_byte:
+            raise ValueError(f"end_byte {end_byte} < start_byte {start_byte}")
+        header = f"bytes={start_byte}-" if end_byte is None else f"bytes={start_byte}-{end_byte}"
+        response = self._session.get(uri, headers={"Range": header})
+        if response.status_code == 416:
+            return b""  # start past EOF: protocol says return the available bytes
+        if response.status_code == 200:
+            raise UnsupportedOperationError(
+                f"server ignored the Range header (answered 200, not 206): {uri}"
+            )
+        self._check_status(uri, response)
+        if response.status_code != 206:
+            raise OndioError(f"unexpected status {response.status_code} for ranged GET: {uri}")
+        return response.content
+
+    def size(self, uri: str) -> int:
+        response = self._session.head(uri, allow_redirects=True)
+        self._check_status(uri, response)
+        length = response.headers.get("Content-Length")
+        if length is None:
+            raise UnsupportedOperationError(f"server reports no Content-Length: {uri}")
+        return int(length)
+
+    def download(self, uri: str, out_path: str | os.PathLike[str]) -> None:
+        dest = Path(out_path)
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        with self._session.get(uri, stream=True) as response:
+            self._check_status(uri, response)
+            with open(dest, "wb") as f:
+                for chunk in response.iter_content(_CHUNK):
+                    f.write(chunk)
+
+    def exists(self, uri: str) -> bool:
+        response = self._session.head(uri, allow_redirects=True)
+        if response.status_code in (405, 501):  # server does not support HEAD
+            with self._session.get(uri, stream=True) as get_response:
+                if get_response.status_code == 404:
+                    return False
+                self._check_status(uri, get_response)
+                return True
+        if response.status_code == 404:
+            return False
+        self._check_status(uri, response)
+        return True
+
+    # --- infeasible over plain HTTP -------------------------------------
+
+    def write(self, uri: str, data: bytes) -> None:
+        raise UnsupportedOperationError(f"write is not supported over HTTP: {uri}")
+
+    def list_files(
+        self,
+        uri_prefix: str,
+        *,
+        recursive: bool = True,
+        max_items: int | None = None,
+        required_prefix: str | None = None,
+        required_ext: str | None = None,
+    ) -> list[str]:
+        raise UnsupportedOperationError(f"list_files is not supported over HTTP: {uri_prefix}")
+
+    def object_count(self, uri_prefix: str, *, required_prefix: str | None = None, required_ext: str | None = None) -> int:
+        raise UnsupportedOperationError(f"object_count is not supported over HTTP: {uri_prefix}")
+
+    def delete(self, uri: str) -> None:
+        raise UnsupportedOperationError(f"delete is not supported over HTTP: {uri}")

ondio/backends/local.py

@@ -0,0 +1,139 @@
+"""Local-filesystem backend."""
+
+from __future__ import annotations
+
+import os
+import shutil 
+
+from pathlib import Path
+from urllib.parse import unquote, urlparse
+
+from ondio.backends.protocol import _matches_filters
+from ondio.types import AuthError, ObjectNotFoundError
+
+def _to_path(uri: str) -> Path:
+    if uri.startswith("file://"):
+        parsed = urlparse(uri)
+        if parsed.netloc not in ("", "localhost"):
+            raise ValueError(f"file:// URIs with a remote host are not supported: {uri!r}")
+        return Path(unquote(parsed.path))
+    return Path(uri)
+
+def _wrap_oserror(uri: str, exc: OSError) -> Exception:
+    if isinstance(exc, PermissionError):
+        return AuthError(f"permission denied: {uri}")
+    return ObjectNotFoundError(f"no such object: {uri}")
+
+
+class LocalBackend:
+    def read(self, uri: str) -> bytes:
+        try:
+            return _to_path(uri).read_bytes()
+        except OSError as exc:
+            raise _wrap_oserror(uri, exc)
+
+
+    def read_range(self, uri: str, start_byte: int, end_byte: int | None) -> bytes:
+        if start_byte < 0:
+            raise ValueError(f"start_byte must be >= 0, got {start_byte}")
+        if end_byte is not None and end_byte < start_byte:
+            raise ValueError(f"end_byte ({end_byte}) < start_byte {start_byte}")
+
+        try:
+            with open(_to_path(uri), "rb") as f:
+                f.seek(start_byte)
+                if end_byte is None:
+                    return f.read()
+                return f.read(end_byte - start_byte + 1)
+        except OSError as exc:
+            raise _wrap_oserror(uri, exc)
+
+    def size(self, uri: str) -> int:
+        path = _to_path(uri)
+        try:
+            if not path.is_file():
+                raise ObjectNotFoundError(f"no such object: {uri}")
+            return path.stat().st_size
+        except OSError as exc:
+            raise _wrap_oserror(uri, exc)
+
+    def write(self, uri: str, data : bytes) -> None:
+        path = _to_path(uri)
+        try:
+            path.parent.mkdir(parents=True, exist_ok=True)
+            path.write_bytes(data)
+        except OSError as exc:
+            raise _wrap_oserror(uri, exc)
+
+
+    def download(self, uri: str, out_path: str | os.PathLike[str]) -> None:
+        src = _to_path(uri)
+        if not src.is_file():
+            raise ObjectNotFoundError(f"no such object: {uri}")
+        dest = Path(out_path)
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copyfile(src, dest)
+
+    def list_files(
+        self,
+        uri_prefix: str,
+        *,
+        recursive: bool = True,
+        max_items: int | None = None,
+        required_prefix: str | None = None,
+        required_ext: str | None = None,
+    ) -> list[str]:
+        as_file_uri = uri_prefix.startswith("file://")
+        matches = (
+            p
+            for p in self._iter_matches(uri_prefix, recursive=recursive)
+            if _matches_filters(p.name, required_prefix, required_ext)
+        )
+        results = sorted(p.as_uri() if as_file_uri else str(p) for p in matches)
+        return results if max_items is None else results[:max_items]
+
+
+    def object_count(
+        self,
+        uri_prefix: str,
+        *,
+        required_prefix: str | None = None,
+        required_ext: str | None = None,
+    ) -> int:
+         return sum(
+            1
+            for path in self._iter_matches(uri_prefix, recursive=True)
+            if _matches_filters(path.name, required_prefix, required_ext)
+        )
+
+    def exists(self, uri: str) -> bool:
+        return _to_path(uri).is_file()
+
+    def delete(self, uri: str) -> None:
+        try:
+            _to_path(uri).unlink(missing_ok=True)
+        except OSError as exc:
+            raise _wrap_oserror(uri, exc) from exc
+
+    def _iter_matches(self, uri_prefix: str, *, recursive: bool):
+        """Files matching a prefix, S3-style: a directory prefix matches everything
+        under it; otherwise the prefix matches any path that starts with it as a
+        string (e.g. /data/chunk_ matches /data/chunk_001.flac)."""
+        base = _to_path(uri_prefix)
+        if base.is_dir():
+            root, filter_prefix = base, None
+        else:
+            root, filter_prefix = base.parent, str(base)
+        if not root.is_dir():
+            return
+        if recursive:
+            candidates = (
+                Path(dirpath) / name
+                for dirpath, _, filenames in os.walk(root)
+                for name in filenames
+            )
+        else:
+            candidates = (p for p in root.iterdir() if p.is_file())
+        for path in candidates:
+            if filter_prefix is None or str(path).startswith(filter_prefix):
+                yield path