pos3 0.1.0rc1__py3-none-any.whl → 0.2.1__py3-none-any.whl

pos3/__init__.py

@@ -17,12 +17,98 @@ from typing import Any
 from urllib.parse import urlparse
 
 import boto3
+from botocore import UNSIGNED
+from botocore.config import Config
 from botocore.exceptions import ClientError
 from tqdm import tqdm
 
 logger = logging.getLogger(__name__)
 
 
+@dataclass(frozen=True)
+class Profile:
+    """Configuration for an S3-compatible endpoint.
+
+    Attributes:
+        local_name: Identifier used in cache path (e.g., 'nebius'). Cannot be '_' (reserved).
+        endpoint: S3 endpoint URL (e.g., 'https://storage.eu-north1.nebius.cloud').
+        public: If True, use anonymous access (no credentials required).
+        region: Optional AWS region name.
+    """
+
+    local_name: str
+    endpoint: str
+    public: bool = False
+    region: str | None = None
+
+    def __post_init__(self):
+        if self.local_name == "_":
+            raise ValueError("Profile local_name cannot be '_' (reserved for default)")
+        if not self.local_name or not all(c.isalnum() or c in "-_" for c in self.local_name):
+            raise ValueError(f"Invalid local_name '{self.local_name}': use only alphanumeric, dash, underscore")
+
+
+_PROFILES: dict[str, Profile] = {}
+
+
+def register_profile(
+    name: str,
+    endpoint: str,
+    public: bool = False,
+    region: str | None = None,
+    local_name: str | None = None,
+) -> None:
+    """Register a named profile for S3 access.
+
+    Creates a Profile with the given parameters. See Profile class for field details.
+    The `local_name` defaults to the profile `name` if not specified.
+    """
+    config = Profile(local_name=local_name or name, endpoint=endpoint, public=public, region=region)
+    existing = _PROFILES.get(name)
+    if existing is not None and existing != config:
+        raise ValueError(f"Profile '{name}' already registered with different config")
+    _PROFILES[name] = config
+
+
+def _resolve_profile(profile: str | Profile | None) -> Profile | None:
+    """Resolve a profile name to a Profile object.
+
+    Args:
+        profile: None, registered profile name (string), or Profile object.
+
+    Returns:
+        Profile object or None.
+
+    Raises:
+        ValueError: If profile is a string that is not registered.
+    """
+    if profile is None or isinstance(profile, Profile):
+        return profile
+    if profile not in _PROFILES:
+        raise ValueError(f"Unknown profile: '{profile}'. Register with pos3.register_profile() first.")
+    return _PROFILES[profile]
+
+
+def _create_s3_client(profile: Profile | None = None):
+    """Create boto3 S3 client, optionally using a profile.
+
+    Args:
+        profile: None (use boto3 defaults) or Profile config.
+    """
+    if profile is None:
+        return boto3.client("s3")
+
+    kwargs: dict[str, Any] = {"endpoint_url": profile.endpoint}
+
+    if profile.region:
+        kwargs["region_name"] = profile.region
+
+    if profile.public:
+        kwargs["config"] = Config(signature_version=UNSIGNED)
+
+    return boto3.client("s3", **kwargs)
+
+
 class _NullTqdm(nullcontext):
     def update(self, *_args: Any, **_kwargs: Any) -> None:  # pragma: no cover - trivial
         pass
@@ -153,11 +239,13 @@ class _Options:
     cache_root: str = "~/.cache/positronic/s3/"
     show_progress: bool = True
     max_workers: int = 10
+    default_profile: Profile | None = None
 
-    def cache_path_for(self, remote: str) -> Path:
+    def cache_path_for(self, remote: str, profile: Profile | None = None) -> Path:
         bucket, key = _parse_s3_url(remote)
         cache_root = Path(self.cache_root).expanduser().resolve()
-        return cache_root / bucket / key
+        local_name = profile.local_name if profile else "_"
+        return cache_root / local_name / bucket / key
 
 
 @dataclass
@@ -166,6 +254,7 @@ class _DownloadRegistration:
     local_path: Path
     delete: bool
     exclude: list[str] | None
+    profile: Profile | None = None
     ready: threading.Event = field(default_factory=threading.Event)
     error: Exception | None = None
 
@@ -177,6 +266,7 @@ class _DownloadRegistration:
             and self.local_path == other.local_path
             and self.delete == other.delete
             and self.exclude == other.exclude
+            and self.profile == other.profile
         )
 
 
@@ -188,6 +278,7 @@ class _UploadRegistration:
     delete: bool
     sync_on_error: bool
     exclude: list[str] | None
+    profile: Profile | None = None
     last_sync: float = 0.0
 
     def __eq__(self, other):
@@ -200,6 +291,7 @@ class _UploadRegistration:
             and self.delete == other.delete
             and self.sync_on_error == other.sync_on_error
             and self.exclude == other.exclude
+            and self.profile == other.profile
         )
 
 
@@ -214,15 +306,28 @@ class _Mirror:
         self.cache_root = Path(self.options.cache_root).expanduser().resolve()
         self.cache_root.mkdir(parents=True, exist_ok=True)
 
-        self.s3_client = boto3.client("s3")
+        self._default_profile = options.default_profile
+        self._clients: dict[Profile | None, Any] = {}
 
-        self._downloads: dict[str, _DownloadRegistration] = {}
-        self._uploads: dict[str, _UploadRegistration] = {}
+        self._downloads: dict[tuple[str, Profile | None], _DownloadRegistration] = {}
+        self._uploads: dict[tuple[str, Profile | None], _UploadRegistration] = {}
         self._lock = threading.RLock()
 
         self._stop_event: threading.Event | None = None
         self._sync_thread: threading.Thread | None = None
 
+    def _effective_profile(self, profile: str | Profile | None) -> Profile | None:
+        """Resolve profile name and substitute default if None."""
+        resolved = _resolve_profile(profile)
+        return resolved if resolved is not None else self._default_profile
+
+    def _get_client(self, profile: Profile | None = None) -> Any:
+        """Get or create S3 client for the given profile."""
+        effective_profile = profile if profile is not None else self._default_profile
+        if effective_profile not in self._clients:
+            self._clients[effective_profile] = _create_s3_client(effective_profile)
+        return self._clients[effective_profile]
+
     @property
     def running(self) -> bool:
         return self._stop_event is not None
@@ -247,6 +352,7 @@ class _Mirror:
         local: str | Path | None,
         delete: bool,
         exclude: list[str] | None = None,
+        profile: str | Profile | None = None,
     ) -> Path:
         """
         Register (and perform if needed) a download from a remote S3 bucket path to a local directory or file.
@@ -257,6 +363,7 @@ class _Mirror:
             local (str | Path | None): Local directory or file destination. If None, uses cache path from options.
             delete (bool): If True, deletes local files not present in S3.
             exclude (list[str] | None): List of glob patterns to exclude from download.
+            profile: S3 profile name or Profile config for custom endpoints.
 
         Returns:
             Path: The canonical local path associated with this download registration.
@@ -265,37 +372,44 @@ class _Mirror:
             FileNotFoundError: If remote is a local path that does not exist.
             ValueError: If download registration conflicts with an existing download or upload or parameters differ.
         """
+        effective_profile = self._effective_profile(profile)
+
         if not _is_s3_path(remote):
             path = Path(remote).expanduser().resolve()
             return path
 
         normalized = _normalize_s3_url(remote)
-        local_path = self.options.cache_path_for(remote) if local is None else Path(local).expanduser().resolve()
+        local_path = (
+            self.options.cache_path_for(remote, effective_profile)
+            if local is None
+            else Path(local).expanduser().resolve()
+        )
         new_registration = _DownloadRegistration(
-            remote=normalized, local_path=local_path, delete=delete, exclude=exclude
+            remote=normalized, local_path=local_path, delete=delete, exclude=exclude, profile=effective_profile
         )
 
         with self._lock:
-            existing = self._downloads.get(normalized)
+            reg_key = (normalized, effective_profile)
+            existing = self._downloads.get(reg_key)
             if existing:
                 if existing != new_registration:
                     raise ValueError(f"Download for '{normalized}' already registered with different parameters")
                 registration = existing
                 need_download = False
             else:
-                self._check_download_conflicts(normalized)
-                self._downloads[normalized] = new_registration
+                self._check_download_conflicts(normalized, effective_profile)
+                self._downloads[reg_key] = new_registration
                 registration = new_registration
                 need_download = True
 
         if need_download:
             try:
-                self._perform_download(normalized, local_path, delete, exclude)
+                self._perform_download(normalized, local_path, delete, exclude, effective_profile)
             except Exception as exc:
                 registration.error = exc
                 registration.ready.set()
                 with self._lock:
-                    self._downloads.pop(normalized, None)
+                    self._downloads.pop(reg_key, None)
                 raise
             else:
                 registration.ready.set()
@@ -314,6 +428,7 @@ class _Mirror:
         delete,
         sync_on_error,
         exclude: list[str] | None = None,
+        profile: str | Profile | None = None,
     ) -> Path:
         """
         Register (and perform if needed) an upload from a local directory or file to a remote S3 bucket path.
@@ -325,6 +440,7 @@ class _Mirror:
             delete (bool): If True, deletes remote files not present locally.
             sync_on_error (bool): If True, attempts to sync files even when encountering errors.
             exclude (list[str] | None): List of glob patterns to exclude from upload.
+            profile: S3 profile name or Profile config for custom endpoints.
 
         Returns:
             Path: The canonical local path associated with this upload registration.
@@ -332,13 +448,19 @@ class _Mirror:
         Raises:
             ValueError: If upload registration conflicts with an existing download or upload or parameters differ.
         """
+        effective_profile = self._effective_profile(profile)
+
         if not _is_s3_path(remote):
             path = Path(remote).expanduser().resolve()
             path.mkdir(parents=True, exist_ok=True)
             return path
 
         normalized = _normalize_s3_url(remote)
-        local_path = self.options.cache_path_for(remote) if local is None else Path(local).expanduser().resolve()
+        local_path = (
+            self.options.cache_path_for(remote, effective_profile)
+            if local is None
+            else Path(local).expanduser().resolve()
+        )
 
         new_registration = _UploadRegistration(
             remote=normalized,
@@ -347,18 +469,20 @@ class _Mirror:
             delete=delete,
             sync_on_error=sync_on_error,
             exclude=exclude,
+            profile=effective_profile,
             last_sync=0,
         )
 
         with self._lock:
-            existing = self._uploads.get(normalized)
+            reg_key = (normalized, effective_profile)
+            existing = self._uploads.get(reg_key)
             if existing:
                 if existing != new_registration:
                     raise ValueError(f"Upload for '{normalized}' already registered with different parameters")
                 return existing.local_path
 
             self._check_upload_conflicts(new_registration)
-            self._uploads[normalized] = new_registration
+            self._uploads[reg_key] = new_registration
             if interval is not None:
                 self._ensure_background_thread_unlocked()
 
@@ -373,18 +497,23 @@ class _Mirror:
         delete_remote: bool,
         sync_on_error: bool,
         exclude: list[str] | None = None,
+        profile: str | Profile | None = None,
     ) -> Path:
-        local_path = self.download(remote, local, delete_local, exclude)
+        # Let download() and upload() handle profile resolution and normalization
+        local_path = self.download(remote, local, delete_local, exclude, profile)
         if not _is_s3_path(remote):
             return local_path
 
         normalized = _normalize_s3_url(remote)
+        effective_profile = self._effective_profile(profile)
         # Unregister the download to allow upload registration for the same remote
-        self._downloads.pop(normalized, None)
-        return self.upload(remote, local_path, interval, delete_remote, sync_on_error, exclude)
+        self._downloads.pop((normalized, effective_profile), None)
+        return self.upload(remote, local_path, interval, delete_remote, sync_on_error, exclude, profile)
 
-    def ls(self, prefix: str, recursive: bool = False) -> list[str]:
+    def ls(self, prefix: str, recursive: bool = False, profile: str | Profile | None = None) -> list[str]:
         """Lists objects under the given prefix, working for both local directories and S3 prefixes."""
+        effective_profile = self._effective_profile(profile)
+
         if _is_s3_path(prefix):
             normalized = _normalize_s3_url(prefix)
             bucket, key = _parse_s3_url(normalized)
@@ -392,7 +521,7 @@ class _Mirror:
             if key:
                 key = key + "/"
             items = []
-            for info in self._scan_s3(bucket, key):
+            for info in self._scan_s3(bucket, key, effective_profile):
                 if info.relative_path:
                     # Skip nested items if not recursive
                     if not recursive and "/" in info.relative_path:
@@ -416,18 +545,19 @@ class _Mirror:
                     items.append(str(display_path.joinpath(Path(info.relative_path))))
             return items
 
-    def _check_download_conflicts(self, candidate: str) -> None:
-        for upload_remote in self._uploads:
-            if _s3_paths_conflict(candidate, upload_remote):
+    def _check_download_conflicts(self, candidate: str, profile: Profile | None) -> None:
+        for (upload_remote, upload_profile), _reg in self._uploads.items():
+            if upload_profile == profile and _s3_paths_conflict(candidate, upload_remote):
                 raise ValueError(f"Conflict: download '{candidate}' overlaps with upload '{upload_remote}'")
 
     def _check_upload_conflicts(self, new_registration) -> None:
         candidate = new_registration.remote
-        for download_remote in self._downloads:
-            if _s3_paths_conflict(candidate, download_remote):
+        candidate_profile = new_registration.profile
+        for (download_remote, download_profile), _reg in self._downloads.items():
+            if download_profile == candidate_profile and _s3_paths_conflict(candidate, download_remote):
                 raise ValueError(f"Conflict: upload '{candidate}' overlaps with download '{download_remote}'")
-        for upload_remote, reg in self._uploads.items():
-            if _s3_paths_conflict(candidate, upload_remote):
+        for (upload_remote, upload_profile), reg in self._uploads.items():
+            if upload_profile == candidate_profile and _s3_paths_conflict(candidate, upload_remote):
                 same_remote = candidate == upload_remote
                 if not same_remote or reg != new_registration:
                     raise ValueError(f"Conflict: upload '{candidate}' overlaps with upload '{upload_remote}'")
@@ -459,7 +589,7 @@ class _Mirror:
         self._sync_uploads(uploads)
 
     def _sync_uploads(self, registrations: Iterable[_UploadRegistration]) -> None:
-        tasks: list[tuple[str, Path, bool, list[str] | None]] = []
+        tasks: list[tuple[str, Path, bool, list[str] | None, Profile | None]] = []
         for registration in registrations:
             if registration.local_path.exists():
                 tasks.append(
@@ -468,32 +598,33 @@ class _Mirror:
                         registration.local_path,
                         registration.delete,
                         registration.exclude,
+                        registration.profile,
                     )
                 )
 
         if not tasks:
             return
 
-        to_put: list[tuple[FileInfo, Path, str, str]] = []
-        to_remove: list[tuple[str, str]] = []
+        to_put: list[tuple[FileInfo, Path, str, str, Profile | None]] = []
+        to_remove: list[tuple[str, str, Profile | None]] = []
         total_bytes = 0
 
-        for remote, local_path, delete, exclude in tasks:
+        for remote, local_path, delete, exclude, profile in tasks:
             logger.debug("Syncing upload: %s from %s (delete=%s)", remote, local_path, delete)
             bucket, prefix = _parse_s3_url(remote)
             to_copy, to_delete = _compute_sync_diff(
                 _filter_fileinfo(_scan_local(local_path), exclude),
-                _filter_fileinfo(self._scan_s3(bucket, prefix), exclude),
+                _filter_fileinfo(self._scan_s3(bucket, prefix, profile), exclude),
             )
 
             for info in to_copy:
                 s3_key = prefix + ("/" + info.relative_path if info.relative_path else "")
-                to_put.append((info, local_path, bucket, s3_key))
+                to_put.append((info, local_path, bucket, s3_key, profile))
                 total_bytes += info.size
 
             for info in to_delete if delete else []:
                 s3_key = prefix + ("/" + info.relative_path if info.relative_path else "")
-                to_remove.append((bucket, s3_key))
+                to_remove.append((bucket, s3_key, profile))
 
         if to_put:
             with (
@@ -501,15 +632,18 @@ class _Mirror:
                 ThreadPoolExecutor(max_workers=self.options.max_workers) as executor,
             ):
                 futures = [
-                    executor.submit(self._put_to_s3, info, local_path, bucket, key, pbar)
-                    for info, local_path, bucket, key in to_put
+                    executor.submit(self._put_to_s3, info, local_path, bucket, key, pbar, profile)
+                    for info, local_path, bucket, key, profile in to_put
                 ]
                 _process_futures(as_completed(futures), "Upload")
 
         if to_remove:
             to_remove_sorted = sorted(to_remove, key=lambda x: x[1].count("/"), reverse=True)
             with ThreadPoolExecutor(max_workers=self.options.max_workers) as executor:
-                futures = [executor.submit(self._remove_from_s3, bucket, key) for bucket, key in to_remove_sorted]
+                futures = [
+                    executor.submit(self._remove_from_s3, bucket, key, profile)
+                    for bucket, key, profile in to_remove_sorted
+                ]
                 iterator = as_completed(futures)
                 if self.options.show_progress:
                     iterator = tqdm(
@@ -519,7 +653,14 @@ class _Mirror:
                     )
                 _process_futures(iterator, "Delete")
 
-    def _perform_download(self, remote: str, local_path: Path, delete: bool, exclude: list[str] | None) -> None:
+    def _perform_download(
+        self,
+        remote: str,
+        local_path: Path,
+        delete: bool,
+        exclude: list[str] | None,
+        profile: Profile | None = None,
+    ) -> None:
         bucket, prefix = _parse_s3_url(remote)
         logger.debug(
             "Performing download: s3://%s/%s to %s (delete=%s)",
@@ -529,7 +670,7 @@ class _Mirror:
             delete,
         )
         to_copy, to_delete = _compute_sync_diff(
-            _filter_fileinfo(self._scan_s3(bucket, prefix), exclude),
+            _filter_fileinfo(self._scan_s3(bucket, prefix, profile), exclude),
             _filter_fileinfo(_scan_local(local_path), exclude),
         )
 
@@ -554,7 +695,7 @@ class _Mirror:
                 self._progress_bar(total_bytes, f"Downloading {remote}") as pbar,
                 ThreadPoolExecutor(max_workers=self.options.max_workers) as executor,
             ):
-                futures = [executor.submit(self._put_locally, *args, pbar) for args in to_put]
+                futures = [executor.submit(self._put_locally, *args, pbar, profile) for args in to_put]
                 _process_futures(as_completed(futures), "Download")
 
         if to_remove:
@@ -565,36 +706,46 @@ class _Mirror:
             for path in iterator:
                 self._remove_locally(path)
 
-    def _list_s3_objects(self, bucket: str, key: str) -> Iterator[dict]:
+    def _list_s3_objects(self, bucket: str, key: str, profile: Profile | None = None) -> Iterator[dict]:
         logger.debug("Listing S3 objects: bucket=%s, key=%s", bucket, key)
-        # Skip head_object for directory-like keys ending with '/'
-        # as we want to list contents, not check if the directory marker exists
-        if not key.endswith("/"):
+        client = self._get_client(profile)
+
+        # Determine the listing prefix - ensure it ends with "/" for directory-like operations
+        # This prevents "droid/recovery" from matching "droid/recovery_towels"
+        list_prefix = key
+
+        # If key doesn't end with "/", try to fetch it as a single object first
+        if key and not key.endswith("/"):
             try:
-                obj = self.s3_client.head_object(Bucket=bucket, Key=key)
+                obj = client.head_object(Bucket=bucket, Key=key)
             except ClientError as exc:
                 error_code = exc.response["Error"]["Code"]
                 if error_code != "404":
                     raise
+                # Not a single file - treat as directory by adding "/"
+                list_prefix = key + "/"
             else:
+                # Found single object
                 logger.debug("Found single object via head_object: %s", key)
                 if "ContentLength" in obj and "Size" not in obj:
                     obj["Size"] = obj["ContentLength"]
                 yield {**obj, "Key": key}
                 return
+        # If key already ends with "/", skip head_object - it's clearly a directory prefix
 
-        paginator = self.s3_client.get_paginator("list_objects_v2")
-        for page in paginator.paginate(Bucket=bucket, Prefix=key):
+        # List with the directory prefix (guaranteed to end with "/")
+        paginator = client.get_paginator("list_objects_v2")
+        for page in paginator.paginate(Bucket=bucket, Prefix=list_prefix):
             objects = page.get("Contents", [])
-            logger.debug("Listed %d objects with prefix %s", len(objects), key)
+            logger.debug("Listed %d objects with prefix %s", len(objects), list_prefix)
             yield from objects
 
-    def _scan_s3(self, bucket: str, prefix: str) -> Iterator[FileInfo]:
+    def _scan_s3(self, bucket: str, prefix: str, profile: Profile | None = None) -> Iterator[FileInfo]:
         logger.debug("Scanning S3: s3://%s/%s", bucket, prefix)
         seen_dirs: set[str] = set()
         has_content = False
 
-        for obj in self._list_s3_objects(bucket, prefix):
+        for obj in self._list_s3_objects(bucket, prefix, profile):
             has_content = True
             key = obj["Key"]
             relative = key[len(prefix) :].lstrip("/")
@@ -625,33 +776,40 @@ class _Mirror:
             return _NullTqdm()
         return tqdm(total=total_bytes, unit="B", unit_scale=True, unit_divisor=1024, desc=desc)
 
-    def _put_to_s3(self, info: FileInfo, local_path: Path, bucket: str, key: str, pbar) -> None:
+    def _put_to_s3(
+        self, info: FileInfo, local_path: Path, bucket: str, key: str, pbar, profile: Profile | None = None
+    ) -> None:
         try:
+            client = self._get_client(profile)
             if info.is_dir:
                 key += "/" if not key.endswith("/") else ""
-                self.s3_client.put_object(Bucket=bucket, Key=key, Body=b"")
+                client.put_object(Bucket=bucket, Key=key, Body=b"")
             else:
                 file_path = local_path / info.relative_path if info.relative_path else local_path
-                self.s3_client.upload_file(str(file_path), bucket, key, Callback=pbar.update)
+                client.upload_file(str(file_path), bucket, key, Callback=pbar.update)
         except Exception as exc:
             logger.error("Failed to put %s to %s/%s: %s", local_path, bucket, key, exc)
             raise
 
-    def _remove_from_s3(self, bucket: str, key: str) -> None:
+    def _remove_from_s3(self, bucket: str, key: str, profile: Profile | None = None) -> None:
         try:
-            self.s3_client.delete_object(Bucket=bucket, Key=key)
+            client = self._get_client(profile)
+            client.delete_object(Bucket=bucket, Key=key)
         except Exception as exc:
             logger.error("Failed to remove %s/%s: %s", bucket, key, exc)
             raise
 
-    def _put_locally(self, info: FileInfo, bucket: str, key: str, local_path: Path, pbar) -> None:
+    def _put_locally(
+        self, info: FileInfo, bucket: str, key: str, local_path: Path, pbar, profile: Profile | None = None
+    ) -> None:
         try:
             target = local_path / info.relative_path if info.relative_path else local_path
             if info.is_dir:
                 target.mkdir(parents=True, exist_ok=True)
             else:
                 target.parent.mkdir(parents=True, exist_ok=True)
-                self.s3_client.download_file(bucket, key, str(target), Callback=pbar.update)
+                client = self._get_client(profile)
+                client.download_file(bucket, key, str(target), Callback=pbar.update)
         except Exception as exc:
             logger.error("Failed to put %s locally: %s", key, exc)
             raise
@@ -672,6 +830,7 @@ def mirror(
     cache_root: str = "~/.cache/positronic/s3/",
     show_progress: bool = True,
     max_workers: int = 10,
+    default_profile: str | Profile | None = None,
 ):
     """
     Context manager that activates the sync environment.
@@ -680,9 +839,16 @@ def mirror(
         cache_root: Base directory for caching downloaded files.
         show_progress: Display tqdm progress bars.
         max_workers: Threads for parallel S3 operations.
+        default_profile: Default S3 profile for all operations in this context.
     """
     global _GLOBAL_ACTIVE_MIRROR
-    options = _Options(cache_root=cache_root, show_progress=show_progress, max_workers=max_workers)
+    resolved_default_profile = _resolve_profile(default_profile)
+    options = _Options(
+        cache_root=cache_root,
+        show_progress=show_progress,
+        max_workers=max_workers,
+        default_profile=resolved_default_profile,
+    )
 
     with _GLOBAL_MIRROR_LOCK:
         if _GLOBAL_ACTIVE_MIRROR is not None:
@@ -712,6 +878,7 @@ def with_mirror(
     cache_root: str = "~/.cache/positronic/s3/",
     show_progress: bool = True,
     max_workers: int = 10,
+    default_profile: str | Profile | None = None,
 ):
     """
     Decorator equivalent of mirror() for wrapping functions.
@@ -721,10 +888,12 @@ def with_mirror(
     def decorator(func):
         @wraps(func)
         def wrapper(*args, **kwargs):
+            # Resolve profile at call time, not decoration time
             with mirror(
                 cache_root=cache_root,
                 show_progress=show_progress,
                 max_workers=max_workers,
+                default_profile=default_profile,
             ):
                 return func(*args, **kwargs)
 
@@ -750,6 +919,7 @@ def download(
     local: str | Path | None = None,
     delete: bool = True,
     exclude: list[str] | None = None,
+    profile: str | Profile | None = None,
 ) -> Path:
     """
     Register a path for download. Ensures local copy matches S3 immediately.
@@ -759,12 +929,13 @@ def download(
         local: Explicit local destination. Defaults to standard cache path.
         delete: If True (default), deletes local files NOT in S3 ("mirror" behavior).
         exclude: List of glob patterns to skip.
+        profile: S3 profile name or Profile config for custom endpoints.
 
     Returns:
         Path to the local directory/file.
     """
     mirror_obj = _require_active_mirror()
-    return mirror_obj.download(remote, local, delete, exclude)
+    return mirror_obj.download(remote, local, delete, exclude, profile)
 
 
 def upload(
@@ -774,6 +945,7 @@ def upload(
     delete: bool = True,
     sync_on_error: bool = False,
     exclude: list[str] | None = None,
+    profile: str | Profile | None = None,
 ) -> Path:
     """
     Register a local path for upload. Uploads on exit and optionally in background.
@@ -784,12 +956,13 @@ def upload(
         interval: Seconds between background syncs. None for exit-only.
         delete: If True (default), deletes S3 files NOT present locally.
         sync_on_error: If True, syncs even if the context exits with an exception.
+        profile: S3 profile name or Profile config for custom endpoints.
 
     Returns:
         Path to the local directory/file.
     """
     mirror_obj = _require_active_mirror()
-    return mirror_obj.upload(remote, local, interval, delete, sync_on_error, exclude)
+    return mirror_obj.upload(remote, local, interval, delete, sync_on_error, exclude, profile)
 
 
 def sync(
@@ -800,6 +973,7 @@ def sync(
     delete_remote: bool = True,
     sync_on_error: bool = False,
     exclude: list[str] | None = None,
+    profile: str | Profile | None = None,
 ) -> Path:
     """
     Bi-directional helper. Performs download() then registers upload().
@@ -807,27 +981,29 @@ def sync(
     Args:
         delete_local: Cleanup local files during download.
         delete_remote: Cleanup remote files during upload.
+        profile: S3 profile name or Profile config for custom endpoints.
 
     Returns:
         Path to the local directory/file.
     """
     mirror_obj = _require_active_mirror()
-    return mirror_obj.sync(remote, local, interval, delete_local, delete_remote, sync_on_error, exclude)
+    return mirror_obj.sync(remote, local, interval, delete_local, delete_remote, sync_on_error, exclude, profile)
 
 
-def ls(prefix: str, recursive: bool = False) -> list[str]:
+def ls(prefix: str, recursive: bool = False, profile: str | Profile | None = None) -> list[str]:
     """
     Lists files/objects in a directory or S3 prefix.
 
     Args:
         prefix: S3 URL or local path.
         recursive: List subdirectories if True.
+        profile: S3 profile name or Profile config for custom endpoints.
 
     Returns:
         List of full S3 URLs or local paths.
     """
     mirror_obj = _require_active_mirror()
-    return mirror_obj.ls(prefix, recursive)
+    return mirror_obj.ls(prefix, recursive, profile)
 
 
-__all__ = ["mirror", "download", "upload", "sync", "ls", "_parse_s3_url"]
+__all__ = ["mirror", "with_mirror", "download", "upload", "sync", "ls", "register_profile", "Profile", "_parse_s3_url"]

{pos3-0.1.0rc1.dist-info → pos3-0.2.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pos3
-Version: 0.1.0rc1
+Version: 0.2.1
 Summary: S3 Simple Sync - Make using S3 as simple as using local files
 Author-email: Positronic Robotics <hi@positronic.ro>
 License: Apache-2.0
@@ -12,6 +12,7 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
+License-File: LICENSE
 Requires-Dist: boto3>=1.26.0
 Requires-Dist: tqdm>=4.65.0
 Provides-Extra: dev
@@ -19,12 +20,15 @@ Requires-Dist: pytest>=7.0; extra == "dev"
 Requires-Dist: pytest-cov; extra == "dev"
 Requires-Dist: ruff; extra == "dev"
 Requires-Dist: pre-commit; extra == "dev"
+Dynamic: license-file
 
 # pos3
 
 **PO**sitronic **S3** — Make using S3 as simple as using local files.
 
-`pos3` provides a Pythonic context manager for syncing directories and files with S3. It is designed for data processing pipelines and machine learning workflows where you want to work with local files but persist data in S3.
+`pos3` provides a Pythonic context manager for syncing directories and files with S3. It is designed for data processing pipelines and machine learning workflows where you need to integrate S3 with code that **only understands local files**.
+
+> The main value of `pos3` is enabling you to pass S3 data to **third-party libraries or legacy scripts** that expect local file paths (e.g., `opencv`, `pandas.read_csv`, or model training scripts). Instead of rewriting their I/O logic to support S3, `pos3` transparently bridges the gap.
 
 ## Core Concepts
 
@@ -144,7 +148,52 @@ Why use `pos3` instead of other Python libraries?
 | **Lifecycle** | **Automated** (Open/Close) | Manual | Manual |
 | **Background Upload** | **Yes** (Non-blocking) | Manual Threading | No (Blocking) |
 | **Local I/O Speed** | **Native** (SSD) | Native | Network Bound (Virtual FS) |
-| **Use Case** | **ML / Pipelines** | App Development | DataFrames / Interactive |
+| **Use Case** | **ML / Pipelines / 3rd Party Code** | App Development | DataFrames / Interactive |
 
 - **vs `boto3`**: `boto3` is the raw AWS SDK. `pos3` wraps it to provide "mirroring" logic, threading, and diffing out of the box.
 - **vs `s3fs`**: `s3fs` treats S3 as a filesystem. `pos3` treats S3 as a persistence layer for your high-speed local storage, ensuring you always get native IO performance.
+
+## Advanced Features
+
+### Profiles
+
+Profiles enable accessing multiple S3-compatible endpoints simultaneously within the same context. This is useful when your workflow combines data from different sources:
+
+```python
+import pos3
+from pos3 import Profile
+
+# Register profiles for different endpoints
+pos3.register_profile('nebius-public',
+    endpoint='https://storage.eu-north1.nebius.cloud',
+    public=True  # anonymous access, no credentials needed
+)
+pos3.register_profile('minio-local',
+    endpoint='http://localhost:9000',
+    region='us-east-1'
+)
+
+# Use multiple profiles in the same context
+with pos3.mirror():
+    # Download public dataset from Nebius
+    dataset = pos3.download('s3://public-data/dataset/', profile='nebius-public')
+
+    # Download private config from local MinIO
+    config = pos3.download('s3://private/config/', profile='minio-local')
+
+    # Upload results to AWS (default boto3 credentials)
+    results = pos3.upload('s3://my-aws-bucket/results/')
+
+    train(dataset, config, results)
+
+# You can also use inline Profile objects without registration
+custom = Profile(local_name='custom', endpoint='https://custom.example.com', public=True)
+with pos3.mirror():
+    data = pos3.download('s3://bucket/path', profile=custom)
+
+# Or set a default profile for the entire context
+with pos3.mirror(default_profile='nebius-public'):
+    data = pos3.download('s3://bucket/path')  # uses nebius-public
+```
+
+Each profile has a `local_name` used in the cache path to keep files from different endpoints separate. When registering profiles, `local_name` defaults to the profile name. The default AWS profile uses `_` as its local name.

pos3-0.2.1.dist-info/RECORD

@@ -0,0 +1,6 @@
+pos3/__init__.py,sha256=PUaRocvB4EnSoX2boF5VcQ6OCREhySCr0DSIOsgo3l0,37730
+pos3-0.2.1.dist-info/licenses/LICENSE,sha256=e815_YqPTxHS3WrNI7dotEuLkgHFAgsf9avLhDYBj9s,11354
+pos3-0.2.1.dist-info/METADATA,sha256=Cmm9JAGaEZF5RkYJEnOR1LDQqWy26HevlD3IRYlVz_s,8637
+pos3-0.2.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+pos3-0.2.1.dist-info/top_level.txt,sha256=JWOpXHz1F6cbH0nfanGWLaozt8RJFRmv5H3eKkxz7e8,5
+pos3-0.2.1.dist-info/RECORD,,

pos3-0.2.1.dist-info/licenses/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2025 Positronic Robotics Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

pos3-0.1.0rc1.dist-info/RECORD

@@ -1,5 +0,0 @@
-pos3/__init__.py,sha256=rxcmD1K5M9zvBqDhIX1guSFxgcm6XOSBhLTbY5wNzkk,30453
-pos3-0.1.0rc1.dist-info/METADATA,sha256=IpXNH8zCxJ1CbTxun7a6uNx1PmylSGGMRFXB2kHEbyU,6620
-pos3-0.1.0rc1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-pos3-0.1.0rc1.dist-info/top_level.txt,sha256=JWOpXHz1F6cbH0nfanGWLaozt8RJFRmv5H3eKkxz7e8,5
-pos3-0.1.0rc1.dist-info/RECORD,,