pos3 0.1.0__py3-none-any.whl → 0.2.0__py3-none-any.whl

pos3/__init__.py

@@ -17,12 +17,98 @@ from typing import Any
 from urllib.parse import urlparse
 
 import boto3
+from botocore import UNSIGNED
+from botocore.config import Config
 from botocore.exceptions import ClientError
 from tqdm import tqdm
 
 logger = logging.getLogger(__name__)
 
 
+@dataclass(frozen=True)
+class Profile:
+    """Configuration for an S3-compatible endpoint.
+
+    Attributes:
+        local_name: Identifier used in cache path (e.g., 'nebius'). Cannot be '_' (reserved).
+        endpoint: S3 endpoint URL (e.g., 'https://storage.eu-north1.nebius.cloud').
+        public: If True, use anonymous access (no credentials required).
+        region: Optional AWS region name.
+    """
+
+    local_name: str
+    endpoint: str
+    public: bool = False
+    region: str | None = None
+
+    def __post_init__(self):
+        if self.local_name == "_":
+            raise ValueError("Profile local_name cannot be '_' (reserved for default)")
+        if not self.local_name or not all(c.isalnum() or c in "-_" for c in self.local_name):
+            raise ValueError(f"Invalid local_name '{self.local_name}': use only alphanumeric, dash, underscore")
+
+
+_PROFILES: dict[str, Profile] = {}
+
+
+def register_profile(
+    name: str,
+    endpoint: str,
+    public: bool = False,
+    region: str | None = None,
+    local_name: str | None = None,
+) -> None:
+    """Register a named profile for S3 access.
+
+    Creates a Profile with the given parameters. See Profile class for field details.
+    The `local_name` defaults to the profile `name` if not specified.
+    """
+    config = Profile(local_name=local_name or name, endpoint=endpoint, public=public, region=region)
+    existing = _PROFILES.get(name)
+    if existing is not None and existing != config:
+        raise ValueError(f"Profile '{name}' already registered with different config")
+    _PROFILES[name] = config
+
+
+def _resolve_profile(profile: str | Profile | None) -> Profile | None:
+    """Resolve a profile name to a Profile object.
+
+    Args:
+        profile: None, registered profile name (string), or Profile object.
+
+    Returns:
+        Profile object or None.
+
+    Raises:
+        ValueError: If profile is a string that is not registered.
+    """
+    if profile is None or isinstance(profile, Profile):
+        return profile
+    if profile not in _PROFILES:
+        raise ValueError(f"Unknown profile: '{profile}'. Register with pos3.register_profile() first.")
+    return _PROFILES[profile]
+
+
+def _create_s3_client(profile: Profile | None = None):
+    """Create boto3 S3 client, optionally using a profile.
+
+    Args:
+        profile: None (use boto3 defaults) or Profile config.
+    """
+    if profile is None:
+        return boto3.client("s3")
+
+    kwargs: dict[str, Any] = {"endpoint_url": profile.endpoint}
+
+    if profile.region:
+        kwargs["region_name"] = profile.region
+
+    if profile.public:
+        kwargs["config"] = Config(signature_version=UNSIGNED)
+
+    return boto3.client("s3", **kwargs)
+
+
 class _NullTqdm(nullcontext):
     def update(self, *_args: Any, **_kwargs: Any) -> None:  # pragma: no cover - trivial
         pass
@@ -153,11 +239,13 @@ class _Options:
     cache_root: str = "~/.cache/positronic/s3/"
     show_progress: bool = True
     max_workers: int = 10
+    default_profile: Profile | None = None
 
-    def cache_path_for(self, remote: str) -> Path:
+    def cache_path_for(self, remote: str, profile: Profile | None = None) -> Path:
         bucket, key = _parse_s3_url(remote)
         cache_root = Path(self.cache_root).expanduser().resolve()
-        return cache_root / bucket / key
+        local_name = profile.local_name if profile else "_"
+        return cache_root / local_name / bucket / key
 
 
 @dataclass
@@ -166,6 +254,7 @@ class _DownloadRegistration:
     local_path: Path
     delete: bool
     exclude: list[str] | None
+    profile: Profile | None = None
     ready: threading.Event = field(default_factory=threading.Event)
     error: Exception | None = None
 
@@ -177,6 +266,7 @@ class _DownloadRegistration:
             and self.local_path == other.local_path
             and self.delete == other.delete
             and self.exclude == other.exclude
+            and self.profile == other.profile
         )
 
 
@@ -188,6 +278,7 @@ class _UploadRegistration:
     delete: bool
     sync_on_error: bool
     exclude: list[str] | None
+    profile: Profile | None = None
     last_sync: float = 0.0
 
     def __eq__(self, other):
@@ -200,6 +291,7 @@ class _UploadRegistration:
             and self.delete == other.delete
             and self.sync_on_error == other.sync_on_error
             and self.exclude == other.exclude
+            and self.profile == other.profile
         )
 
 
@@ -214,15 +306,28 @@ class _Mirror:
         self.cache_root = Path(self.options.cache_root).expanduser().resolve()
         self.cache_root.mkdir(parents=True, exist_ok=True)
 
-        self.s3_client = boto3.client("s3")
+        self._default_profile = options.default_profile
+        self._clients: dict[Profile | None, Any] = {}
 
-        self._downloads: dict[str, _DownloadRegistration] = {}
-        self._uploads: dict[str, _UploadRegistration] = {}
+        self._downloads: dict[tuple[str, Profile | None], _DownloadRegistration] = {}
+        self._uploads: dict[tuple[str, Profile | None], _UploadRegistration] = {}
         self._lock = threading.RLock()
 
         self._stop_event: threading.Event | None = None
         self._sync_thread: threading.Thread | None = None
 
+    def _effective_profile(self, profile: str | Profile | None) -> Profile | None:
+        """Resolve profile name and substitute default if None."""
+        resolved = _resolve_profile(profile)
+        return resolved if resolved is not None else self._default_profile
+
+    def _get_client(self, profile: Profile | None = None) -> Any:
+        """Get or create S3 client for the given profile."""
+        effective_profile = profile if profile is not None else self._default_profile
+        if effective_profile not in self._clients:
+            self._clients[effective_profile] = _create_s3_client(effective_profile)
+        return self._clients[effective_profile]
+
     @property
     def running(self) -> bool:
         return self._stop_event is not None
@@ -247,6 +352,7 @@ class _Mirror:
         local: str | Path | None,
         delete: bool,
         exclude: list[str] | None = None,
+        profile: str | Profile | None = None,
     ) -> Path:
         """
         Register (and perform if needed) a download from a remote S3 bucket path to a local directory or file.
@@ -257,6 +363,7 @@ class _Mirror:
             local (str | Path | None): Local directory or file destination. If None, uses cache path from options.
             delete (bool): If True, deletes local files not present in S3.
             exclude (list[str] | None): List of glob patterns to exclude from download.
+            profile: S3 profile name or Profile config for custom endpoints.
 
         Returns:
             Path: The canonical local path associated with this download registration.
@@ -265,37 +372,44 @@ class _Mirror:
             FileNotFoundError: If remote is a local path that does not exist.
             ValueError: If download registration conflicts with an existing download or upload or parameters differ.
         """
+        effective_profile = self._effective_profile(profile)
+
         if not _is_s3_path(remote):
             path = Path(remote).expanduser().resolve()
             return path
 
         normalized = _normalize_s3_url(remote)
-        local_path = self.options.cache_path_for(remote) if local is None else Path(local).expanduser().resolve()
+        local_path = (
+            self.options.cache_path_for(remote, effective_profile)
+            if local is None
+            else Path(local).expanduser().resolve()
+        )
         new_registration = _DownloadRegistration(
-            remote=normalized, local_path=local_path, delete=delete, exclude=exclude
+            remote=normalized, local_path=local_path, delete=delete, exclude=exclude, profile=effective_profile
         )
 
         with self._lock:
-            existing = self._downloads.get(normalized)
+            reg_key = (normalized, effective_profile)
+            existing = self._downloads.get(reg_key)
             if existing:
                 if existing != new_registration:
                     raise ValueError(f"Download for '{normalized}' already registered with different parameters")
                 registration = existing
                 need_download = False
             else:
-                self._check_download_conflicts(normalized)
-                self._downloads[normalized] = new_registration
+                self._check_download_conflicts(normalized, effective_profile)
+                self._downloads[reg_key] = new_registration
                 registration = new_registration
                 need_download = True
 
         if need_download:
             try:
-                self._perform_download(normalized, local_path, delete, exclude)
+                self._perform_download(normalized, local_path, delete, exclude, effective_profile)
             except Exception as exc:
                 registration.error = exc
                 registration.ready.set()
                 with self._lock:
-                    self._downloads.pop(normalized, None)
+                    self._downloads.pop(reg_key, None)
                 raise
             else:
                 registration.ready.set()
@@ -314,6 +428,7 @@ class _Mirror:
         delete,
         sync_on_error,
         exclude: list[str] | None = None,
+        profile: str | Profile | None = None,
     ) -> Path:
         """
         Register (and perform if needed) an upload from a local directory or file to a remote S3 bucket path.
@@ -325,6 +440,7 @@ class _Mirror:
             delete (bool): If True, deletes remote files not present locally.
             sync_on_error (bool): If True, attempts to sync files even when encountering errors.
             exclude (list[str] | None): List of glob patterns to exclude from upload.
+            profile: S3 profile name or Profile config for custom endpoints.
 
         Returns:
             Path: The canonical local path associated with this upload registration.
@@ -332,13 +448,19 @@ class _Mirror:
         Raises:
             ValueError: If upload registration conflicts with an existing download or upload or parameters differ.
         """
+        effective_profile = self._effective_profile(profile)
+
         if not _is_s3_path(remote):
             path = Path(remote).expanduser().resolve()
             path.mkdir(parents=True, exist_ok=True)
             return path
 
         normalized = _normalize_s3_url(remote)
-        local_path = self.options.cache_path_for(remote) if local is None else Path(local).expanduser().resolve()
+        local_path = (
+            self.options.cache_path_for(remote, effective_profile)
+            if local is None
+            else Path(local).expanduser().resolve()
+        )
 
         new_registration = _UploadRegistration(
             remote=normalized,
@@ -347,18 +469,20 @@ class _Mirror:
             delete=delete,
             sync_on_error=sync_on_error,
             exclude=exclude,
+            profile=effective_profile,
             last_sync=0,
         )
 
         with self._lock:
-            existing = self._uploads.get(normalized)
+            reg_key = (normalized, effective_profile)
+            existing = self._uploads.get(reg_key)
             if existing:
                 if existing != new_registration:
                     raise ValueError(f"Upload for '{normalized}' already registered with different parameters")
                 return existing.local_path
 
             self._check_upload_conflicts(new_registration)
-            self._uploads[normalized] = new_registration
+            self._uploads[reg_key] = new_registration
             if interval is not None:
                 self._ensure_background_thread_unlocked()
 
@@ -373,18 +497,23 @@ class _Mirror:
         delete_remote: bool,
         sync_on_error: bool,
         exclude: list[str] | None = None,
+        profile: str | Profile | None = None,
     ) -> Path:
-        local_path = self.download(remote, local, delete_local, exclude)
+        # Let download() and upload() handle profile resolution and normalization
+        local_path = self.download(remote, local, delete_local, exclude, profile)
         if not _is_s3_path(remote):
             return local_path
 
         normalized = _normalize_s3_url(remote)
+        effective_profile = self._effective_profile(profile)
         # Unregister the download to allow upload registration for the same remote
-        self._downloads.pop(normalized, None)
-        return self.upload(remote, local_path, interval, delete_remote, sync_on_error, exclude)
+        self._downloads.pop((normalized, effective_profile), None)
+        return self.upload(remote, local_path, interval, delete_remote, sync_on_error, exclude, profile)
 
-    def ls(self, prefix: str, recursive: bool = False) -> list[str]:
+    def ls(self, prefix: str, recursive: bool = False, profile: str | Profile | None = None) -> list[str]:
         """Lists objects under the given prefix, working for both local directories and S3 prefixes."""
+        effective_profile = self._effective_profile(profile)
+
         if _is_s3_path(prefix):
             normalized = _normalize_s3_url(prefix)
             bucket, key = _parse_s3_url(normalized)
@@ -392,7 +521,7 @@ class _Mirror:
             if key:
                 key = key + "/"
             items = []
-            for info in self._scan_s3(bucket, key):
+            for info in self._scan_s3(bucket, key, effective_profile):
                 if info.relative_path:
                     # Skip nested items if not recursive
                     if not recursive and "/" in info.relative_path:
@@ -416,18 +545,19 @@ class _Mirror:
                     items.append(str(display_path.joinpath(Path(info.relative_path))))
             return items
 
-    def _check_download_conflicts(self, candidate: str) -> None:
-        for upload_remote in self._uploads:
-            if _s3_paths_conflict(candidate, upload_remote):
+    def _check_download_conflicts(self, candidate: str, profile: Profile | None) -> None:
+        for (upload_remote, upload_profile), _reg in self._uploads.items():
+            if upload_profile == profile and _s3_paths_conflict(candidate, upload_remote):
                 raise ValueError(f"Conflict: download '{candidate}' overlaps with upload '{upload_remote}'")
 
     def _check_upload_conflicts(self, new_registration) -> None:
         candidate = new_registration.remote
-        for download_remote in self._downloads:
-            if _s3_paths_conflict(candidate, download_remote):
+        candidate_profile = new_registration.profile
+        for (download_remote, download_profile), _reg in self._downloads.items():
+            if download_profile == candidate_profile and _s3_paths_conflict(candidate, download_remote):
                 raise ValueError(f"Conflict: upload '{candidate}' overlaps with download '{download_remote}'")
-        for upload_remote, reg in self._uploads.items():
-            if _s3_paths_conflict(candidate, upload_remote):
+        for (upload_remote, upload_profile), reg in self._uploads.items():
+            if upload_profile == candidate_profile and _s3_paths_conflict(candidate, upload_remote):
                 same_remote = candidate == upload_remote
                 if not same_remote or reg != new_registration:
                     raise ValueError(f"Conflict: upload '{candidate}' overlaps with upload '{upload_remote}'")
@@ -459,7 +589,7 @@ class _Mirror:
         self._sync_uploads(uploads)
 
     def _sync_uploads(self, registrations: Iterable[_UploadRegistration]) -> None:
-        tasks: list[tuple[str, Path, bool, list[str] | None]] = []
+        tasks: list[tuple[str, Path, bool, list[str] | None, Profile | None]] = []
         for registration in registrations:
             if registration.local_path.exists():
                 tasks.append(
@@ -468,32 +598,33 @@ class _Mirror:
                         registration.local_path,
                         registration.delete,
                         registration.exclude,
+                        registration.profile,
                     )
                 )
 
         if not tasks:
             return
 
-        to_put: list[tuple[FileInfo, Path, str, str]] = []
-        to_remove: list[tuple[str, str]] = []
+        to_put: list[tuple[FileInfo, Path, str, str, Profile | None]] = []
+        to_remove: list[tuple[str, str, Profile | None]] = []
         total_bytes = 0
 
-        for remote, local_path, delete, exclude in tasks:
+        for remote, local_path, delete, exclude, profile in tasks:
             logger.debug("Syncing upload: %s from %s (delete=%s)", remote, local_path, delete)
             bucket, prefix = _parse_s3_url(remote)
             to_copy, to_delete = _compute_sync_diff(
                 _filter_fileinfo(_scan_local(local_path), exclude),
-                _filter_fileinfo(self._scan_s3(bucket, prefix), exclude),
+                _filter_fileinfo(self._scan_s3(bucket, prefix, profile), exclude),
             )
 
             for info in to_copy:
                 s3_key = prefix + ("/" + info.relative_path if info.relative_path else "")
-                to_put.append((info, local_path, bucket, s3_key))
+                to_put.append((info, local_path, bucket, s3_key, profile))
                 total_bytes += info.size
 
             for info in to_delete if delete else []:
                 s3_key = prefix + ("/" + info.relative_path if info.relative_path else "")
-                to_remove.append((bucket, s3_key))
+                to_remove.append((bucket, s3_key, profile))
 
         if to_put:
             with (
@@ -501,15 +632,18 @@ class _Mirror:
                 ThreadPoolExecutor(max_workers=self.options.max_workers) as executor,
             ):
                 futures = [
-                    executor.submit(self._put_to_s3, info, local_path, bucket, key, pbar)
-                    for info, local_path, bucket, key in to_put
+                    executor.submit(self._put_to_s3, info, local_path, bucket, key, pbar, profile)
+                    for info, local_path, bucket, key, profile in to_put
                 ]
                 _process_futures(as_completed(futures), "Upload")
 
         if to_remove:
             to_remove_sorted = sorted(to_remove, key=lambda x: x[1].count("/"), reverse=True)
             with ThreadPoolExecutor(max_workers=self.options.max_workers) as executor:
-                futures = [executor.submit(self._remove_from_s3, bucket, key) for bucket, key in to_remove_sorted]
+                futures = [
+                    executor.submit(self._remove_from_s3, bucket, key, profile)
+                    for bucket, key, profile in to_remove_sorted
+                ]
                 iterator = as_completed(futures)
                 if self.options.show_progress:
                     iterator = tqdm(
@@ -519,7 +653,14 @@ class _Mirror:
                     )
                 _process_futures(iterator, "Delete")
 
-    def _perform_download(self, remote: str, local_path: Path, delete: bool, exclude: list[str] | None) -> None:
+    def _perform_download(
+        self,
+        remote: str,
+        local_path: Path,
+        delete: bool,
+        exclude: list[str] | None,
+        profile: Profile | None = None,
+    ) -> None:
         bucket, prefix = _parse_s3_url(remote)
         logger.debug(
             "Performing download: s3://%s/%s to %s (delete=%s)",
@@ -529,7 +670,7 @@ class _Mirror:
             delete,
         )
         to_copy, to_delete = _compute_sync_diff(
-            _filter_fileinfo(self._scan_s3(bucket, prefix), exclude),
+            _filter_fileinfo(self._scan_s3(bucket, prefix, profile), exclude),
             _filter_fileinfo(_scan_local(local_path), exclude),
         )
 
@@ -554,7 +695,7 @@ class _Mirror:
                 self._progress_bar(total_bytes, f"Downloading {remote}") as pbar,
                 ThreadPoolExecutor(max_workers=self.options.max_workers) as executor,
             ):
-                futures = [executor.submit(self._put_locally, *args, pbar) for args in to_put]
+                futures = [executor.submit(self._put_locally, *args, pbar, profile) for args in to_put]
                 _process_futures(as_completed(futures), "Download")
 
         if to_remove:
@@ -565,13 +706,14 @@ class _Mirror:
             for path in iterator:
                 self._remove_locally(path)
 
-    def _list_s3_objects(self, bucket: str, key: str) -> Iterator[dict]:
+    def _list_s3_objects(self, bucket: str, key: str, profile: Profile | None = None) -> Iterator[dict]:
         logger.debug("Listing S3 objects: bucket=%s, key=%s", bucket, key)
+        client = self._get_client(profile)
         # Skip head_object for directory-like keys ending with '/'
         # as we want to list contents, not check if the directory marker exists
         if not key.endswith("/"):
             try:
-                obj = self.s3_client.head_object(Bucket=bucket, Key=key)
+                obj = client.head_object(Bucket=bucket, Key=key)
             except ClientError as exc:
                 error_code = exc.response["Error"]["Code"]
                 if error_code != "404":
@@ -583,18 +725,18 @@ class _Mirror:
                 yield {**obj, "Key": key}
                 return
 
-        paginator = self.s3_client.get_paginator("list_objects_v2")
+        paginator = client.get_paginator("list_objects_v2")
         for page in paginator.paginate(Bucket=bucket, Prefix=key):
             objects = page.get("Contents", [])
             logger.debug("Listed %d objects with prefix %s", len(objects), key)
             yield from objects
 
-    def _scan_s3(self, bucket: str, prefix: str) -> Iterator[FileInfo]:
+    def _scan_s3(self, bucket: str, prefix: str, profile: Profile | None = None) -> Iterator[FileInfo]:
         logger.debug("Scanning S3: s3://%s/%s", bucket, prefix)
         seen_dirs: set[str] = set()
         has_content = False
 
-        for obj in self._list_s3_objects(bucket, prefix):
+        for obj in self._list_s3_objects(bucket, prefix, profile):
             has_content = True
             key = obj["Key"]
             relative = key[len(prefix) :].lstrip("/")
@@ -625,33 +767,40 @@ class _Mirror:
             return _NullTqdm()
         return tqdm(total=total_bytes, unit="B", unit_scale=True, unit_divisor=1024, desc=desc)
 
-    def _put_to_s3(self, info: FileInfo, local_path: Path, bucket: str, key: str, pbar) -> None:
+    def _put_to_s3(
+        self, info: FileInfo, local_path: Path, bucket: str, key: str, pbar, profile: Profile | None = None
+    ) -> None:
         try:
+            client = self._get_client(profile)
             if info.is_dir:
                 key += "/" if not key.endswith("/") else ""
-                self.s3_client.put_object(Bucket=bucket, Key=key, Body=b"")
+                client.put_object(Bucket=bucket, Key=key, Body=b"")
             else:
                 file_path = local_path / info.relative_path if info.relative_path else local_path
-                self.s3_client.upload_file(str(file_path), bucket, key, Callback=pbar.update)
+                client.upload_file(str(file_path), bucket, key, Callback=pbar.update)
         except Exception as exc:
             logger.error("Failed to put %s to %s/%s: %s", local_path, bucket, key, exc)
             raise
 
-    def _remove_from_s3(self, bucket: str, key: str) -> None:
+    def _remove_from_s3(self, bucket: str, key: str, profile: Profile | None = None) -> None:
         try:
-            self.s3_client.delete_object(Bucket=bucket, Key=key)
+            client = self._get_client(profile)
+            client.delete_object(Bucket=bucket, Key=key)
         except Exception as exc:
             logger.error("Failed to remove %s/%s: %s", bucket, key, exc)
             raise
 
-    def _put_locally(self, info: FileInfo, bucket: str, key: str, local_path: Path, pbar) -> None:
+    def _put_locally(
+        self, info: FileInfo, bucket: str, key: str, local_path: Path, pbar, profile: Profile | None = None
+    ) -> None:
         try:
             target = local_path / info.relative_path if info.relative_path else local_path
             if info.is_dir:
                 target.mkdir(parents=True, exist_ok=True)
             else:
                 target.parent.mkdir(parents=True, exist_ok=True)
-                self.s3_client.download_file(bucket, key, str(target), Callback=pbar.update)
+                client = self._get_client(profile)
+                client.download_file(bucket, key, str(target), Callback=pbar.update)
         except Exception as exc:
             logger.error("Failed to put %s locally: %s", key, exc)
             raise
@@ -672,6 +821,7 @@ def mirror(
     cache_root: str = "~/.cache/positronic/s3/",
     show_progress: bool = True,
     max_workers: int = 10,
+    default_profile: str | Profile | None = None,
 ):
     """
     Context manager that activates the sync environment.
@@ -680,9 +830,16 @@ def mirror(
         cache_root: Base directory for caching downloaded files.
         show_progress: Display tqdm progress bars.
         max_workers: Threads for parallel S3 operations.
+        default_profile: Default S3 profile for all operations in this context.
     """
     global _GLOBAL_ACTIVE_MIRROR
-    options = _Options(cache_root=cache_root, show_progress=show_progress, max_workers=max_workers)
+    resolved_default_profile = _resolve_profile(default_profile)
+    options = _Options(
+        cache_root=cache_root,
+        show_progress=show_progress,
+        max_workers=max_workers,
+        default_profile=resolved_default_profile,
+    )
 
     with _GLOBAL_MIRROR_LOCK:
         if _GLOBAL_ACTIVE_MIRROR is not None:
@@ -712,6 +869,7 @@ def with_mirror(
     cache_root: str = "~/.cache/positronic/s3/",
     show_progress: bool = True,
     max_workers: int = 10,
+    default_profile: str | Profile | None = None,
 ):
     """
     Decorator equivalent of mirror() for wrapping functions.
@@ -721,10 +879,12 @@ def with_mirror(
     def decorator(func):
         @wraps(func)
         def wrapper(*args, **kwargs):
+            # Resolve profile at call time, not decoration time
             with mirror(
                 cache_root=cache_root,
                 show_progress=show_progress,
                 max_workers=max_workers,
+                default_profile=default_profile,
             ):
                 return func(*args, **kwargs)
 
@@ -750,6 +910,7 @@ def download(
     local: str | Path | None = None,
     delete: bool = True,
     exclude: list[str] | None = None,
+    profile: str | Profile | None = None,
 ) -> Path:
     """
     Register a path for download. Ensures local copy matches S3 immediately.
@@ -759,12 +920,13 @@ def download(
         local: Explicit local destination. Defaults to standard cache path.
         delete: If True (default), deletes local files NOT in S3 ("mirror" behavior).
         exclude: List of glob patterns to skip.
+        profile: S3 profile name or Profile config for custom endpoints.
 
     Returns:
         Path to the local directory/file.
     """
     mirror_obj = _require_active_mirror()
-    return mirror_obj.download(remote, local, delete, exclude)
+    return mirror_obj.download(remote, local, delete, exclude, profile)
 
 
 def upload(
@@ -774,6 +936,7 @@ def upload(
     delete: bool = True,
     sync_on_error: bool = False,
     exclude: list[str] | None = None,
+    profile: str | Profile | None = None,
 ) -> Path:
     """
     Register a local path for upload. Uploads on exit and optionally in background.
@@ -784,12 +947,13 @@ def upload(
         interval: Seconds between background syncs. None for exit-only.
         delete: If True (default), deletes S3 files NOT present locally.
         sync_on_error: If True, syncs even if the context exits with an exception.
+        profile: S3 profile name or Profile config for custom endpoints.
 
     Returns:
         Path to the local directory/file.
     """
     mirror_obj = _require_active_mirror()
-    return mirror_obj.upload(remote, local, interval, delete, sync_on_error, exclude)
+    return mirror_obj.upload(remote, local, interval, delete, sync_on_error, exclude, profile)
 
 
 def sync(
@@ -800,6 +964,7 @@ def sync(
     delete_remote: bool = True,
     sync_on_error: bool = False,
     exclude: list[str] | None = None,
+    profile: str | Profile | None = None,
 ) -> Path:
     """
     Bi-directional helper. Performs download() then registers upload().
@@ -807,27 +972,29 @@ def sync(
     Args:
         delete_local: Cleanup local files during download.
         delete_remote: Cleanup remote files during upload.
+        profile: S3 profile name or Profile config for custom endpoints.
 
     Returns:
         Path to the local directory/file.
     """
     mirror_obj = _require_active_mirror()
-    return mirror_obj.sync(remote, local, interval, delete_local, delete_remote, sync_on_error, exclude)
+    return mirror_obj.sync(remote, local, interval, delete_local, delete_remote, sync_on_error, exclude, profile)
 
 
-def ls(prefix: str, recursive: bool = False) -> list[str]:
+def ls(prefix: str, recursive: bool = False, profile: str | Profile | None = None) -> list[str]:
     """
     Lists files/objects in a directory or S3 prefix.
 
     Args:
         prefix: S3 URL or local path.
         recursive: List subdirectories if True.
+        profile: S3 profile name or Profile config for custom endpoints.
 
     Returns:
         List of full S3 URLs or local paths.
     """
     mirror_obj = _require_active_mirror()
-    return mirror_obj.ls(prefix, recursive)
+    return mirror_obj.ls(prefix, recursive, profile)
 
 
-__all__ = ["mirror", "download", "upload", "sync", "ls", "_parse_s3_url"]
+__all__ = ["mirror", "with_mirror", "download", "upload", "sync", "ls", "register_profile", "Profile", "_parse_s3_url"]

{pos3-0.1.0.dist-info → pos3-0.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pos3
-Version: 0.1.0
+Version: 0.2.0
 Summary: S3 Simple Sync - Make using S3 as simple as using local files
 Author-email: Positronic Robotics <hi@positronic.ro>
 License: Apache-2.0
@@ -12,6 +12,7 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
+License-File: LICENSE
 Requires-Dist: boto3>=1.26.0
 Requires-Dist: tqdm>=4.65.0
 Provides-Extra: dev
@@ -19,6 +20,7 @@ Requires-Dist: pytest>=7.0; extra == "dev"
 Requires-Dist: pytest-cov; extra == "dev"
 Requires-Dist: ruff; extra == "dev"
 Requires-Dist: pre-commit; extra == "dev"
+Dynamic: license-file
 
 # pos3
 
@@ -150,3 +152,48 @@ Why use `pos3` instead of other Python libraries?
 
 - **vs `boto3`**: `boto3` is the raw AWS SDK. `pos3` wraps it to provide "mirroring" logic, threading, and diffing out of the box.
 - **vs `s3fs`**: `s3fs` treats S3 as a filesystem. `pos3` treats S3 as a persistence layer for your high-speed local storage, ensuring you always get native IO performance.
+
+## Advanced Features
+
+### Profiles
+
+Profiles enable accessing multiple S3-compatible endpoints simultaneously within the same context. This is useful when your workflow combines data from different sources:
+
+```python
+import pos3
+from pos3 import Profile
+
+# Register profiles for different endpoints
+pos3.register_profile('nebius-public',
+    endpoint='https://storage.eu-north1.nebius.cloud',
+    public=True  # anonymous access, no credentials needed
+)
+pos3.register_profile('minio-local',
+    endpoint='http://localhost:9000',
+    region='us-east-1'
+)
+
+# Use multiple profiles in the same context
+with pos3.mirror():
+    # Download public dataset from Nebius
+    dataset = pos3.download('s3://public-data/dataset/', profile='nebius-public')
+
+    # Download private config from local MinIO
+    config = pos3.download('s3://private/config/', profile='minio-local')
+
+    # Upload results to AWS (default boto3 credentials)
+    results = pos3.upload('s3://my-aws-bucket/results/')
+
+    train(dataset, config, results)
+
+# You can also use inline Profile objects without registration
+custom = Profile(local_name='custom', endpoint='https://custom.example.com', public=True)
+with pos3.mirror():
+    data = pos3.download('s3://bucket/path', profile=custom)
+
+# Or set a default profile for the entire context
+with pos3.mirror(default_profile='nebius-public'):
+    data = pos3.download('s3://bucket/path')  # uses nebius-public
+```
+
+Each profile has a `local_name` used in the cache path to keep files from different endpoints separate. When registering profiles, `local_name` defaults to the profile name. The default AWS profile uses `_` as its local name.

pos3-0.2.0.dist-info/RECORD

@@ -0,0 +1,6 @@
+pos3/__init__.py,sha256=ByRzIJ3ggRKFU6j8HtzDv7D3RW_6NZkbjp3etoMilys,37261
+pos3-0.2.0.dist-info/licenses/LICENSE,sha256=e815_YqPTxHS3WrNI7dotEuLkgHFAgsf9avLhDYBj9s,11354
+pos3-0.2.0.dist-info/METADATA,sha256=LF6o86VvogFSbkcsfy39yVmf-37IjbZdqJZ3guem4CA,8637
+pos3-0.2.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+pos3-0.2.0.dist-info/top_level.txt,sha256=JWOpXHz1F6cbH0nfanGWLaozt8RJFRmv5H3eKkxz7e8,5
+pos3-0.2.0.dist-info/RECORD,,

pos3-0.2.0.dist-info/licenses/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2025 Positronic Robotics Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

pos3-0.1.0.dist-info/RECORD

@@ -1,5 +0,0 @@
-pos3/__init__.py,sha256=rxcmD1K5M9zvBqDhIX1guSFxgcm6XOSBhLTbY5wNzkk,30453
-pos3-0.1.0.dist-info/METADATA,sha256=9XUDlEQjdkmJt7WX0PDeThq5X1ZplpEnJnNRrj03V7M,6940
-pos3-0.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-pos3-0.1.0.dist-info/top_level.txt,sha256=JWOpXHz1F6cbH0nfanGWLaozt8RJFRmv5H3eKkxz7e8,5
-pos3-0.1.0.dist-info/RECORD,,