pos3 0.1.0__py3-none-any.whl

pos3/__init__.py

@@ -0,0 +1,833 @@
+"""Global S3 mirror API with dynamic registration."""
+
+from __future__ import annotations
+
+import logging
+import shutil
+import threading
+import time
+from collections.abc import Iterable, Iterator
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from contextlib import contextmanager, nullcontext
+from contextvars import ContextVar
+from dataclasses import dataclass, field
+from functools import wraps
+from pathlib import Path, PurePosixPath
+from typing import Any
+from urllib.parse import urlparse
+
+import boto3
+from botocore.exceptions import ClientError
+from tqdm import tqdm
+
+logger = logging.getLogger(__name__)
+
+
+class _NullTqdm(nullcontext):
+    def update(self, *_args: Any, **_kwargs: Any) -> None:  # pragma: no cover - trivial
+        pass
+
+    def __enter__(self) -> _NullTqdm:
+        return self
+
+
+def _parse_s3_url(s3_url: str) -> tuple[str, str]:
+    parsed = urlparse(s3_url)
+    if parsed.scheme != "s3":
+        raise ValueError(f"Not an S3 URL: {s3_url}")
+    return parsed.netloc, parsed.path.lstrip("/")
+
+
+def _normalize_s3_url(s3_url: str) -> str:
+    bucket, key = _parse_s3_url(s3_url)
+    key = key.strip("/")
+    return f"s3://{bucket}/{key}" if key else f"s3://{bucket}"
+
+
+def _is_s3_path(path: str) -> bool:
+    return path.startswith("s3://")
+
+
+def _s3_paths_conflict(left: str, right: str) -> bool:
+    left_norm = left.rstrip("/")
+    right_norm = right.rstrip("/")
+    if left_norm == right_norm:
+        return True
+    return left_norm.startswith(right_norm + "/") or right_norm.startswith(left_norm + "/")
+
+
+def _process_futures(futures, operation: str) -> None:
+    for future in futures:
+        try:
+            future.result()
+        except Exception as exc:
+            logger.error("%s failed: %s", operation, exc)
+
+
+@dataclass(frozen=True)
+class FileInfo:
+    """Represents a file or directory with metadata for sync operations."""
+
+    relative_path: str  # Relative path from root (empty string for root file/dir)
+    size: int  # File size in bytes, 0 for directories
+    is_dir: bool  # True if this represents a directory
+
+
+def _scan_local(path: Path) -> Iterator[FileInfo]:
+    if not path.exists():
+        return
+
+    base = path
+    stack = [path]
+    # path => relative
+    while stack:
+        p = stack.pop()
+        if p.is_symlink():
+            continue
+
+        relative = p.relative_to(base).as_posix() if p != base else ""
+        if p.is_dir():
+            # Always yield directories, including the root (relative_path='')
+            yield FileInfo(relative_path=relative, size=0, is_dir=True)
+            stack.extend(p.iterdir())
+        else:
+            yield FileInfo(relative_path=relative, size=p.stat().st_size, is_dir=False)
+
+
+def _filter_fileinfo(fileinfo_iter: Iterator[FileInfo], exclude: list[str] | None) -> Iterator[FileInfo]:
+    """
+    Filter FileInfo objects based on glob patterns.
+
+    Args:
+        fileinfo_iter: Iterator of FileInfo objects to filter
+        exclude: List of glob patterns to exclude (supports *, **, ?, [...])
+
+    Yields:
+        FileInfo objects that don't match any exclude patterns
+    """
+    if not exclude:
+        yield from fileinfo_iter
+        return
+    excluded_dirs: set[str] = set()
+
+    for info in fileinfo_iter:
+        # Skip if any parent directory was excluded
+        if any(info.relative_path == ed or info.relative_path.startswith(ed + "/") for ed in excluded_dirs):
+            continue
+
+        # Check if this path matches any exclude pattern
+        path = PurePosixPath(info.relative_path) if info.relative_path else PurePosixPath(".")
+        if any(path.match(pattern) for pattern in exclude):
+            if info.is_dir:
+                excluded_dirs.add(info.relative_path)
+        else:
+            yield info
+
+
+def _compute_sync_diff(source: Iterator[FileInfo], target: Iterator[FileInfo]) -> tuple[list[FileInfo], list[FileInfo]]:
+    source_map: dict[str, FileInfo] = {info.relative_path: info for info in source}
+    target_map: dict[str, FileInfo] = {info.relative_path: info for info in target}
+
+    to_copy, to_delete = [], []
+
+    for relative_path, source_info in source_map.items():
+        target_info = target_map.get(relative_path)
+
+        if target_info is None:
+            to_copy.append(source_info)
+        elif source_info.is_dir != target_info.is_dir:
+            to_delete.append(target_info)
+            to_copy.append(source_info)
+        elif not source_info.is_dir and source_info.size != target_info.size:
+            to_copy.append(source_info)
+
+    for relative_path, target_info in target_map.items():
+        if relative_path not in source_map:
+            to_delete.append(target_info)
+
+    return to_copy, to_delete
+
+
+@dataclass
+class _Options:
+    cache_root: str = "~/.cache/positronic/s3/"
+    show_progress: bool = True
+    max_workers: int = 10
+
+    def cache_path_for(self, remote: str) -> Path:
+        bucket, key = _parse_s3_url(remote)
+        cache_root = Path(self.cache_root).expanduser().resolve()
+        return cache_root / bucket / key
+
+
+@dataclass
+class _DownloadRegistration:
+    remote: str
+    local_path: Path
+    delete: bool
+    exclude: list[str] | None
+    ready: threading.Event = field(default_factory=threading.Event)
+    error: Exception | None = None
+
+    def __eq__(self, other):
+        if not isinstance(other, _DownloadRegistration):
+            return False
+        return (
+            self.remote == other.remote
+            and self.local_path == other.local_path
+            and self.delete == other.delete
+            and self.exclude == other.exclude
+        )
+
+
+@dataclass
+class _UploadRegistration:
+    remote: str
+    local_path: Path
+    interval: int | None
+    delete: bool
+    sync_on_error: bool
+    exclude: list[str] | None
+    last_sync: float = 0.0
+
+    def __eq__(self, other):
+        if not isinstance(other, _UploadRegistration):
+            return False
+        return (
+            self.remote == other.remote
+            and self.local_path == other.local_path
+            and self.interval == other.interval
+            and self.delete == other.delete
+            and self.sync_on_error == other.sync_on_error
+            and self.exclude == other.exclude
+        )
+
+
+_ACTIVE_MIRROR: ContextVar[_Mirror | None] = ContextVar("_ACTIVE_MIRROR", default=None)
+_GLOBAL_MIRROR_LOCK = threading.RLock()
+_GLOBAL_ACTIVE_MIRROR: _Mirror | None = None
+
+
+class _Mirror:
+    def __init__(self, options: _Options):
+        self.options = options
+        self.cache_root = Path(self.options.cache_root).expanduser().resolve()
+        self.cache_root.mkdir(parents=True, exist_ok=True)
+
+        self.s3_client = boto3.client("s3")
+
+        self._downloads: dict[str, _DownloadRegistration] = {}
+        self._uploads: dict[str, _UploadRegistration] = {}
+        self._lock = threading.RLock()
+
+        self._stop_event: threading.Event | None = None
+        self._sync_thread: threading.Thread | None = None
+
+    @property
+    def running(self) -> bool:
+        return self._stop_event is not None
+
+    def start(self) -> None:
+        if not self.running:
+            self._stop_event = threading.Event()
+
+    def stop(self, had_error: bool = False) -> None:
+        if self.running:
+            self._stop_event.set()
+
+            if self._sync_thread:
+                self._sync_thread.join(timeout=60)
+                self._sync_thread = None
+            self._final_sync(had_error=had_error)
+            self._stop_event = None
+
+    def download(
+        self,
+        remote: str,
+        local: str | Path | None,
+        delete: bool,
+        exclude: list[str] | None = None,
+    ) -> Path:
+        """
+        Register (and perform if needed) a download from a remote S3 bucket path to a local directory or file.
+
+        Args:
+            remote (str): Source S3 URL (e.g., "s3://bucket/key/prefix") or local path.
+                If a local path is provided, it is validated and returned directly.
+            local (str | Path | None): Local directory or file destination. If None, uses cache path from options.
+            delete (bool): If True, deletes local files not present in S3.
+            exclude (list[str] | None): List of glob patterns to exclude from download.
+
+        Returns:
+            Path: The canonical local path associated with this download registration.
+
+        Raises:
+            FileNotFoundError: If remote is a local path that does not exist.
+            ValueError: If download registration conflicts with an existing download or upload or parameters differ.
+        """
+        if not _is_s3_path(remote):
+            path = Path(remote).expanduser().resolve()
+            return path
+
+        normalized = _normalize_s3_url(remote)
+        local_path = self.options.cache_path_for(remote) if local is None else Path(local).expanduser().resolve()
+        new_registration = _DownloadRegistration(
+            remote=normalized, local_path=local_path, delete=delete, exclude=exclude
+        )
+
+        with self._lock:
+            existing = self._downloads.get(normalized)
+            if existing:
+                if existing != new_registration:
+                    raise ValueError(f"Download for '{normalized}' already registered with different parameters")
+                registration = existing
+                need_download = False
+            else:
+                self._check_download_conflicts(normalized)
+                self._downloads[normalized] = new_registration
+                registration = new_registration
+                need_download = True
+
+        if need_download:
+            try:
+                self._perform_download(normalized, local_path, delete, exclude)
+            except Exception as exc:
+                registration.error = exc
+                registration.ready.set()
+                with self._lock:
+                    self._downloads.pop(normalized, None)
+                raise
+            else:
+                registration.ready.set()
+        else:
+            registration.ready.wait()
+            if registration.error is not None:
+                raise registration.error
+
+        return local_path
+
+    def upload(
+        self,
+        remote,
+        local,
+        interval,
+        delete,
+        sync_on_error,
+        exclude: list[str] | None = None,
+    ) -> Path:
+        """
+        Register (and perform if needed) an upload from a local directory or file to a remote S3 bucket path.
+
+        Args:
+            remote (str): Destination S3 URL (e.g., "s3://bucket/key/prefix")
+            local (str | Path | None): Local directory or file to upload. If None, determines default from options.
+            interval (int | None): If set, enables periodic background uploads (seconds between syncs).
+            delete (bool): If True, deletes remote files not present locally.
+            sync_on_error (bool): If True, attempts to sync files even when encountering errors.
+            exclude (list[str] | None): List of glob patterns to exclude from upload.
+
+        Returns:
+            Path: The canonical local path associated with this upload registration.
+
+        Raises:
+            ValueError: If upload registration conflicts with an existing download or upload or parameters differ.
+        """
+        if not _is_s3_path(remote):
+            path = Path(remote).expanduser().resolve()
+            path.mkdir(parents=True, exist_ok=True)
+            return path
+
+        normalized = _normalize_s3_url(remote)
+        local_path = self.options.cache_path_for(remote) if local is None else Path(local).expanduser().resolve()
+
+        new_registration = _UploadRegistration(
+            remote=normalized,
+            local_path=local_path,
+            interval=interval,
+            delete=delete,
+            sync_on_error=sync_on_error,
+            exclude=exclude,
+            last_sync=0,
+        )
+
+        with self._lock:
+            existing = self._uploads.get(normalized)
+            if existing:
+                if existing != new_registration:
+                    raise ValueError(f"Upload for '{normalized}' already registered with different parameters")
+                return existing.local_path
+
+            self._check_upload_conflicts(new_registration)
+            self._uploads[normalized] = new_registration
+            if interval is not None:
+                self._ensure_background_thread_unlocked()
+
+        return local_path
+
+    def sync(
+        self,
+        remote: str,
+        local: str | Path | None,
+        interval: int | None,
+        delete_local: bool,
+        delete_remote: bool,
+        sync_on_error: bool,
+        exclude: list[str] | None = None,
+    ) -> Path:
+        local_path = self.download(remote, local, delete_local, exclude)
+        if not _is_s3_path(remote):
+            return local_path
+
+        normalized = _normalize_s3_url(remote)
+        # Unregister the download to allow upload registration for the same remote
+        self._downloads.pop(normalized, None)
+        return self.upload(remote, local_path, interval, delete_remote, sync_on_error, exclude)
+
+    def ls(self, prefix: str, recursive: bool = False) -> list[str]:
+        """Lists objects under the given prefix, working for both local directories and S3 prefixes."""
+        if _is_s3_path(prefix):
+            normalized = _normalize_s3_url(prefix)
+            bucket, key = _parse_s3_url(normalized)
+            # Ensure directory-like listing by appending '/' to avoid spurious prefix matches
+            if key:
+                key = key + "/"
+            items = []
+            for info in self._scan_s3(bucket, key):
+                if info.relative_path:
+                    # Skip nested items if not recursive
+                    if not recursive and "/" in info.relative_path:
+                        continue
+                    # Reconstruct the full S3 key
+                    if key:
+                        s3_key = key.rstrip("/") + "/" + info.relative_path
+                    else:
+                        s3_key = info.relative_path
+                    items.append(f"s3://{bucket}/{s3_key}")
+            return items
+        else:
+            display_path = Path(prefix).expanduser()
+            scan_path = display_path.resolve()
+            items = []
+            for info in _scan_local(scan_path):
+                if info.relative_path:
+                    # Skip nested items if not recursive
+                    if not recursive and "/" in info.relative_path:
+                        continue
+                    items.append(str(display_path.joinpath(Path(info.relative_path))))
+            return items
+
+    def _check_download_conflicts(self, candidate: str) -> None:
+        for upload_remote in self._uploads:
+            if _s3_paths_conflict(candidate, upload_remote):
+                raise ValueError(f"Conflict: download '{candidate}' overlaps with upload '{upload_remote}'")
+
+    def _check_upload_conflicts(self, new_registration) -> None:
+        candidate = new_registration.remote
+        for download_remote in self._downloads:
+            if _s3_paths_conflict(candidate, download_remote):
+                raise ValueError(f"Conflict: upload '{candidate}' overlaps with download '{download_remote}'")
+        for upload_remote, reg in self._uploads.items():
+            if _s3_paths_conflict(candidate, upload_remote):
+                same_remote = candidate == upload_remote
+                if not same_remote or reg != new_registration:
+                    raise ValueError(f"Conflict: upload '{candidate}' overlaps with upload '{upload_remote}'")
+
+    def _ensure_background_thread_unlocked(self) -> None:
+        assert self.running, "The mirror must be started before performing any uploads"
+        if not self._sync_thread or not self._sync_thread.is_alive():
+            thread = threading.Thread(target=self._background_worker, name="positronic-s3-sync", daemon=True)
+            thread.start()
+            self._sync_thread = thread
+
+    def _background_worker(self) -> None:
+        while not self._stop_event.wait(1):
+            now = time.monotonic()
+            due: list[_UploadRegistration] = []
+            with self._lock:
+                for registration in self._uploads.values():
+                    if registration.interval is not None and now - registration.last_sync >= registration.interval:
+                        registration.last_sync = now
+                        due.append(registration)
+
+            self._sync_uploads(due)
+
+    def _final_sync(self, had_error: bool = False) -> None:
+        with self._lock:
+            uploads = list(self._uploads.values())
+        if had_error:
+            uploads = [u for u in uploads if u.sync_on_error]
+        self._sync_uploads(uploads)
+
+    def _sync_uploads(self, registrations: Iterable[_UploadRegistration]) -> None:
+        tasks: list[tuple[str, Path, bool, list[str] | None]] = []
+        for registration in registrations:
+            if registration.local_path.exists():
+                tasks.append(
+                    (
+                        registration.remote,
+                        registration.local_path,
+                        registration.delete,
+                        registration.exclude,
+                    )
+                )
+
+        if not tasks:
+            return
+
+        to_put: list[tuple[FileInfo, Path, str, str]] = []
+        to_remove: list[tuple[str, str]] = []
+        total_bytes = 0
+
+        for remote, local_path, delete, exclude in tasks:
+            logger.debug("Syncing upload: %s from %s (delete=%s)", remote, local_path, delete)
+            bucket, prefix = _parse_s3_url(remote)
+            to_copy, to_delete = _compute_sync_diff(
+                _filter_fileinfo(_scan_local(local_path), exclude),
+                _filter_fileinfo(self._scan_s3(bucket, prefix), exclude),
+            )
+
+            for info in to_copy:
+                s3_key = prefix + ("/" + info.relative_path if info.relative_path else "")
+                to_put.append((info, local_path, bucket, s3_key))
+                total_bytes += info.size
+
+            for info in to_delete if delete else []:
+                s3_key = prefix + ("/" + info.relative_path if info.relative_path else "")
+                to_remove.append((bucket, s3_key))
+
+        if to_put:
+            with (
+                self._progress_bar(total_bytes, f"Uploading {remote}") as pbar,
+                ThreadPoolExecutor(max_workers=self.options.max_workers) as executor,
+            ):
+                futures = [
+                    executor.submit(self._put_to_s3, info, local_path, bucket, key, pbar)
+                    for info, local_path, bucket, key in to_put
+                ]
+                _process_futures(as_completed(futures), "Upload")
+
+        if to_remove:
+            to_remove_sorted = sorted(to_remove, key=lambda x: x[1].count("/"), reverse=True)
+            with ThreadPoolExecutor(max_workers=self.options.max_workers) as executor:
+                futures = [executor.submit(self._remove_from_s3, bucket, key) for bucket, key in to_remove_sorted]
+                iterator = as_completed(futures)
+                if self.options.show_progress:
+                    iterator = tqdm(
+                        iterator,
+                        total=len(to_remove_sorted),
+                        desc=f"Deleting in {remote}",
+                    )
+                _process_futures(iterator, "Delete")
+
+    def _perform_download(self, remote: str, local_path: Path, delete: bool, exclude: list[str] | None) -> None:
+        bucket, prefix = _parse_s3_url(remote)
+        logger.debug(
+            "Performing download: s3://%s/%s to %s (delete=%s)",
+            bucket,
+            prefix,
+            local_path,
+            delete,
+        )
+        to_copy, to_delete = _compute_sync_diff(
+            _filter_fileinfo(self._scan_s3(bucket, prefix), exclude),
+            _filter_fileinfo(_scan_local(local_path), exclude),
+        )
+
+        to_put: list[tuple[FileInfo, str, str, Path]] = []
+        to_remove: list[Path] = []
+        total_bytes = 0
+
+        for info in to_copy:
+            s3_key = prefix + ("/" + info.relative_path if info.relative_path else "")
+            to_put.append((info, bucket, s3_key, local_path))
+            total_bytes += info.size
+
+        if delete:
+            logger.debug("Will delete %d local items not in S3", len(to_delete))
+        for info in to_delete if delete else []:
+            target = local_path / info.relative_path if info.relative_path else local_path
+            logger.debug("Marking for local deletion: %s", target)
+            to_remove.append(target)
+
+        if to_put:
+            with (
+                self._progress_bar(total_bytes, f"Downloading {remote}") as pbar,
+                ThreadPoolExecutor(max_workers=self.options.max_workers) as executor,
+            ):
+                futures = [executor.submit(self._put_locally, *args, pbar) for args in to_put]
+                _process_futures(as_completed(futures), "Download")
+
+        if to_remove:
+            to_remove_sorted = sorted(to_remove, key=lambda x: len(x.parts), reverse=True)
+            iterator = to_remove_sorted
+            if self.options.show_progress:
+                iterator = tqdm(iterator, desc=f"Deleting in {remote}")
+            for path in iterator:
+                self._remove_locally(path)
+
+    def _list_s3_objects(self, bucket: str, key: str) -> Iterator[dict]:
+        logger.debug("Listing S3 objects: bucket=%s, key=%s", bucket, key)
+        # Skip head_object for directory-like keys ending with '/'
+        # as we want to list contents, not check if the directory marker exists
+        if not key.endswith("/"):
+            try:
+                obj = self.s3_client.head_object(Bucket=bucket, Key=key)
+            except ClientError as exc:
+                error_code = exc.response["Error"]["Code"]
+                if error_code != "404":
+                    raise
+            else:
+                logger.debug("Found single object via head_object: %s", key)
+                if "ContentLength" in obj and "Size" not in obj:
+                    obj["Size"] = obj["ContentLength"]
+                yield {**obj, "Key": key}
+                return
+
+        paginator = self.s3_client.get_paginator("list_objects_v2")
+        for page in paginator.paginate(Bucket=bucket, Prefix=key):
+            objects = page.get("Contents", [])
+            logger.debug("Listed %d objects with prefix %s", len(objects), key)
+            yield from objects
+
+    def _scan_s3(self, bucket: str, prefix: str) -> Iterator[FileInfo]:
+        logger.debug("Scanning S3: s3://%s/%s", bucket, prefix)
+        seen_dirs: set[str] = set()
+        has_content = False
+
+        for obj in self._list_s3_objects(bucket, prefix):
+            has_content = True
+            key = obj["Key"]
+            relative = key[len(prefix) :].lstrip("/")
+
+            if key.endswith("/"):
+                relative = relative.rstrip("/")
+                if relative:
+                    yield FileInfo(relative_path=relative, size=0, is_dir=True)
+                    seen_dirs.add(relative)
+            else:
+                yield FileInfo(relative_path=relative, size=obj["Size"], is_dir=False)
+
+                if "/" in relative:
+                    parts = relative.split("/")
+                    for i in range(len(parts) - 1):
+                        dir_path = "/".join(parts[: i + 1])
+                        if dir_path and dir_path not in seen_dirs:
+                            yield FileInfo(relative_path=dir_path, size=0, is_dir=True)
+                            seen_dirs.add(dir_path)
+
+        if has_content:
+            yield FileInfo(
+                relative_path="", size=0, is_dir=True
+            )  # Yield root directory marker for symmetry with _scan_local
+
+    def _progress_bar(self, total_bytes: int, desc: str):
+        if not self.options.show_progress:
+            return _NullTqdm()
+        return tqdm(total=total_bytes, unit="B", unit_scale=True, unit_divisor=1024, desc=desc)
+
+    def _put_to_s3(self, info: FileInfo, local_path: Path, bucket: str, key: str, pbar) -> None:
+        try:
+            if info.is_dir:
+                key += "/" if not key.endswith("/") else ""
+                self.s3_client.put_object(Bucket=bucket, Key=key, Body=b"")
+            else:
+                file_path = local_path / info.relative_path if info.relative_path else local_path
+                self.s3_client.upload_file(str(file_path), bucket, key, Callback=pbar.update)
+        except Exception as exc:
+            logger.error("Failed to put %s to %s/%s: %s", local_path, bucket, key, exc)
+            raise
+
+    def _remove_from_s3(self, bucket: str, key: str) -> None:
+        try:
+            self.s3_client.delete_object(Bucket=bucket, Key=key)
+        except Exception as exc:
+            logger.error("Failed to remove %s/%s: %s", bucket, key, exc)
+            raise
+
+    def _put_locally(self, info: FileInfo, bucket: str, key: str, local_path: Path, pbar) -> None:
+        try:
+            target = local_path / info.relative_path if info.relative_path else local_path
+            if info.is_dir:
+                target.mkdir(parents=True, exist_ok=True)
+            else:
+                target.parent.mkdir(parents=True, exist_ok=True)
+                self.s3_client.download_file(bucket, key, str(target), Callback=pbar.update)
+        except Exception as exc:
+            logger.error("Failed to put %s locally: %s", key, exc)
+            raise
+
+    def _remove_locally(self, path: Path) -> None:
+        try:
+            if path.is_dir():
+                shutil.rmtree(path)
+            else:
+                path.unlink()
+        except Exception as exc:
+            logger.error("Failed to remove %s: %s", path, exc)
+            raise
+
+
+@contextmanager
+def mirror(
+    cache_root: str = "~/.cache/positronic/s3/",
+    show_progress: bool = True,
+    max_workers: int = 10,
+):
+    """
+    Context manager that activates the sync environment.
+
+    Args:
+        cache_root: Base directory for caching downloaded files.
+        show_progress: Display tqdm progress bars.
+        max_workers: Threads for parallel S3 operations.
+    """
+    global _GLOBAL_ACTIVE_MIRROR
+    options = _Options(cache_root=cache_root, show_progress=show_progress, max_workers=max_workers)
+
+    with _GLOBAL_MIRROR_LOCK:
+        if _GLOBAL_ACTIVE_MIRROR is not None:
+            raise RuntimeError("Mirror already active")
+
+        mirror_obj = _Mirror(options)
+        mirror_obj.start()
+        _GLOBAL_ACTIVE_MIRROR = mirror_obj
+
+    token = _ACTIVE_MIRROR.set(mirror_obj)
+    had_error = False
+    try:
+        yield
+    except Exception:
+        had_error = True
+        raise
+    finally:
+        try:
+            mirror_obj.stop(had_error=had_error)
+        finally:
+            with _GLOBAL_MIRROR_LOCK:
+                _GLOBAL_ACTIVE_MIRROR = None
+            _ACTIVE_MIRROR.reset(token)
+
+
+def with_mirror(
+    cache_root: str = "~/.cache/positronic/s3/",
+    show_progress: bool = True,
+    max_workers: int = 10,
+):
+    """
+    Decorator equivalent of mirror() for wrapping functions.
+    See mirror() for argument details.
+    """
+
+    def decorator(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            with mirror(
+                cache_root=cache_root,
+                show_progress=show_progress,
+                max_workers=max_workers,
+            ):
+                return func(*args, **kwargs)
+
+        return wrapper
+
+    return decorator
+
+
+def _require_active_mirror() -> _Mirror:
+    mirror_obj = _ACTIVE_MIRROR.get()
+    if mirror_obj is not None:
+        return mirror_obj
+
+    global _GLOBAL_ACTIVE_MIRROR
+    if _GLOBAL_ACTIVE_MIRROR is not None:
+        return _GLOBAL_ACTIVE_MIRROR
+
+    raise RuntimeError("No active mirror context")
+
+
+def download(
+    remote: str,
+    local: str | Path | None = None,
+    delete: bool = True,
+    exclude: list[str] | None = None,
+) -> Path:
+    """
+    Register a path for download. Ensures local copy matches S3 immediately.
+
+    Args:
+        remote: S3 URL or local path.
+        local: Explicit local destination. Defaults to standard cache path.
+        delete: If True (default), deletes local files NOT in S3 ("mirror" behavior).
+        exclude: List of glob patterns to skip.
+
+    Returns:
+        Path to the local directory/file.
+    """
+    mirror_obj = _require_active_mirror()
+    return mirror_obj.download(remote, local, delete, exclude)
+
+
+def upload(
+    remote: str,
+    local: str | Path | None = None,
+    interval: int | None = 300,
+    delete: bool = True,
+    sync_on_error: bool = False,
+    exclude: list[str] | None = None,
+) -> Path:
+    """
+    Register a local path for upload. Uploads on exit and optionally in background.
+
+    Args:
+        remote: Destination S3 URL.
+        local: Local source path. Auto-resolved from cache path if None.
+        interval: Seconds between background syncs. None for exit-only.
+        delete: If True (default), deletes S3 files NOT present locally.
+        sync_on_error: If True, syncs even if the context exits with an exception.
+
+    Returns:
+        Path to the local directory/file.
+    """
+    mirror_obj = _require_active_mirror()
+    return mirror_obj.upload(remote, local, interval, delete, sync_on_error, exclude)
+
+
+def sync(
+    remote: str,
+    local: str | Path | None = None,
+    interval: int | None = 300,
+    delete_local: bool = True,
+    delete_remote: bool = True,
+    sync_on_error: bool = False,
+    exclude: list[str] | None = None,
+) -> Path:
+    """
+    Bi-directional helper. Performs download() then registers upload().
+
+    Args:
+        delete_local: Cleanup local files during download.
+        delete_remote: Cleanup remote files during upload.
+
+    Returns:
+        Path to the local directory/file.
+    """
+    mirror_obj = _require_active_mirror()
+    return mirror_obj.sync(remote, local, interval, delete_local, delete_remote, sync_on_error, exclude)
+
+
+def ls(prefix: str, recursive: bool = False) -> list[str]:
+    """
+    Lists files/objects in a directory or S3 prefix.
+
+    Args:
+        prefix: S3 URL or local path.
+        recursive: List subdirectories if True.
+
+    Returns:
+        List of full S3 URLs or local paths.
+    """
+    mirror_obj = _require_active_mirror()
+    return mirror_obj.ls(prefix, recursive)
+
+
+__all__ = ["mirror", "download", "upload", "sync", "ls", "_parse_s3_url"]

pos3-0.1.0.dist-info/METADATA

@@ -0,0 +1,152 @@
+Metadata-Version: 2.4
+Name: pos3
+Version: 0.1.0
+Summary: S3 Simple Sync - Make using S3 as simple as using local files
+Author-email: Positronic Robotics <hi@positronic.ro>
+License: Apache-2.0
+Project-URL: Homepage, https://github.com/Positronic-Robotics/pos3
+Project-URL: Repository, https://github.com/Positronic-Robotics/pos3
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: boto3>=1.26.0
+Requires-Dist: tqdm>=4.65.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: pytest-cov; extra == "dev"
+Requires-Dist: ruff; extra == "dev"
+Requires-Dist: pre-commit; extra == "dev"
+
+# pos3
+
+**PO**sitronic **S3** — Make using S3 as simple as using local files.
+
+`pos3` provides a Pythonic context manager for syncing directories and files with S3. It is designed for data processing pipelines and machine learning workflows where you need to integrate S3 with code that **only understands local files**.
+
+> The main value of `pos3` is enabling you to pass S3 data to **third-party libraries or legacy scripts** that expect local file paths (e.g., `opencv`, `pandas.read_csv`, or model training scripts). Instead of rewriting their I/O logic to support S3, `pos3` transparently bridges the gap.
+
+## Core Concepts
+
+- **Context Manager**: All operations run within a `with pos3.mirror():` block.
+    - **Enter**: Initializes the sync environment (threads, cache).
+    - **Body**: You explicitly call `pos3.download()` to fetch files and `pos3.upload()` to register outputs.
+    - **Exit**: Uploads registered output paths (mirroring local to S3).
+- **Lazy & Efficient**: Only transfers files that have changed (based on size/presence).
+- **Local Paths**: All API calls return a `pathlib.Path` to the local file/directory. If you pass a local path instead of an S3 URL, it is passed through unchanged (no copy).
+- **Background Sync**: Can optionally upload changes in the background (e.g., every 60s) for long-running jobs.
+
+## Quick Start
+
+The primary API is the `pos3.mirror()` context manager.
+
+```python
+import pos3
+
+# 1. Start the context
+with pos3.mirror(cache_root='~/.cache/positronic/s3'):
+
+    # 2. Download Input
+    #    - Downloads s3://bucket/data to cache
+    #    - Deletes local files that don't exist in S3 (mirroring)
+    #    - Returns local Path object
+    dataset_path = pos3.download('s3://bucket/data')
+
+    # 3. Sync Output (Resume & Upload)
+    #    - Downloads existing checkpoints (to resume)
+    #    - Registers path for background uploads
+    checkpoints_path = pos3.sync('s3://bucket/ckpt', interval=60, delete_remote=False)
+
+    # 4. Upload Logs (Write-only)
+    #    - Creates local directory
+    #    - Uploads new files to S3 on exit/interval
+    logs_path = pos3.upload('s3://bucket/logs', interval=30)
+
+    # 5. Use standard local file paths
+    print(f"Reading from {dataset_path}")      # -> ~/.cache/positronic/s3/bucket/data
+    print(f"Writing to {checkpoints_path}")    # -> ~/.cache/positronic/s3/bucket/ckpt
+    print(f"Logging to {logs_path}")           # -> ~/.cache/positronic/s3/bucket/logs
+
+    train(dataset_path, checkpoints_path, logs_path)
+```
+
+## API Guide
+
+> **Note**: All operational methods (`download`, `upload`, `sync`, `ls`) must be called within an active `pos3.mirror()` context. Calling them outside will raise a `RuntimeError`.
+
+### `pos3.mirror(...)` / `@pos3.with_mirror(...)`
+
+Context manager (or decorator) that activates the sync environment.
+
+**Parameters:**
+- `cache_root` (default: `'~/.cache/positronic/s3/'`): Base directory for caching downloaded files.
+- `show_progress` (default: `True`): Display tqdm progress bars.
+- `max_workers` (default: `10`): Threads for parallel S3 operations.
+
+**Decorator Example:**
+
+```python
+@pos3.with_mirror(cache_root='/tmp/cache')
+def main():
+    # Only works when called!
+    data_path = pos3.download('s3://bucket/data')
+    train(data_path)
+
+if __name__ == "__main__":
+    main()
+```
+
+### `pos3.download(remote, local=None, delete=True, exclude=None)`
+
+Registers a path for download. Ensures local copy matches S3 immediately.
+- `remote`: S3 URL (e.g., `s3://bucket/key`) or local path.
+- `local`: Explicit local destination. Defaults to standard cache path.
+- `delete`: If `True` (default), deletes local files NOT in S3 ("mirror" behavior).
+- `exclude`: List of glob patterns to skip.
+
+**Returns**: `pathlib.Path` to the local directory/file.
+
+### `pos3.upload(remote, local=None, interval=300, delete=True, sync_on_error=False, exclude=None)`
+
+Registers a local path for upload. Uploads on exit and optionally in background.
+- `remote`: Destination S3 URL.
+- `local`: Local source path. Auto-resolved from cache path if `None`.
+- `interval`: Seconds between background syncs. `None` for exit-only.
+- `delete`: If `True` (default), deletes S3 files NOT present locally.
+- `sync_on_error`: If `True`, syncs even if the context exits with an exception.
+
+**Returns**: `pathlib.Path` to the local directory/file.
+
+### `pos3.sync(remote, local=None, interval=300, delete_local=True, delete_remote=True, sync_on_error=False, exclude=None)`
+
+Bi-directional helper. Performs `download()` then registers `upload()`. Useful for jobs that work on existing files, like when you resume training from a checkpoint.
+- `delete_local`: Cleanup local files during download.
+- `delete_remote`: Cleanup remote files during upload. carefully consider setting to `False` when resuming jobs to avoid deleting history.
+
+**Returns**: `pathlib.Path` to the local directory/file.
+
+### `pos3.ls(prefix, recursive=False)`
+
+Lists files/objects in a directory or S3 prefix.
+- `prefix`: S3 URL or local path.
+- `recursive`: List subdirectories if `True`.
+
+**Returns**: List of full S3 URLs or local paths.
+
+## Comparison with Libraries
+
+Why use `pos3` instead of other Python libraries?
+
+| Feature | `pos3` | `boto3` | `s3fs` / `fsspec` |
+| :--- | :--- | :--- | :--- |
+| **Abstraction Level** | **High** (Context Manager) | **Low** (API Client) | **Medium** (File System) |
+| **Sync Logic** | **Built-in** (Differential) | Manual Implementation | `put`/`get` (Recursive) |
+| **Lifecycle** | **Automated** (Open/Close) | Manual | Manual |
+| **Background Upload** | **Yes** (Non-blocking) | Manual Threading | No (Blocking) |
+| **Local I/O Speed** | **Native** (SSD) | Native | Network Bound (Virtual FS) |
+| **Use Case** | **ML / Pipelines / 3rd Party Code** | App Development | DataFrames / Interactive |
+
+- **vs `boto3`**: `boto3` is the raw AWS SDK. `pos3` wraps it to provide "mirroring" logic, threading, and diffing out of the box.
+- **vs `s3fs`**: `s3fs` treats S3 as a filesystem. `pos3` treats S3 as a persistence layer for your high-speed local storage, ensuring you always get native IO performance.

pos3-0.1.0.dist-info/RECORD

@@ -0,0 +1,5 @@
+pos3/__init__.py,sha256=rxcmD1K5M9zvBqDhIX1guSFxgcm6XOSBhLTbY5wNzkk,30453
+pos3-0.1.0.dist-info/METADATA,sha256=9XUDlEQjdkmJt7WX0PDeThq5X1ZplpEnJnNRrj03V7M,6940
+pos3-0.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+pos3-0.1.0.dist-info/top_level.txt,sha256=JWOpXHz1F6cbH0nfanGWLaozt8RJFRmv5H3eKkxz7e8,5
+pos3-0.1.0.dist-info/RECORD,,

pos3-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

pos3-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+pos3