coiled 1.122.0__tar.gz → 1.124.0__tar.gz

{coiled-1.122.0 → coiled-1.124.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: coiled
-Version: 1.122.0
+Version: 1.124.0
 Summary: Python client for coiled.io dask clusters
 Project-URL: Homepage, https://coiled.io
 Maintainer-email: Coiled <info@coiled.io>

{coiled-1.122.0 → coiled-1.124.0}/coiled/filestore.py

@@ -135,13 +135,24 @@ def upload_to_filestore_with_ui(fs, local_dir):
                 },
             ])
 
-            upload_urls = FilestoreManager.get_signed_upload_urls(fs["id"], files_for_upload=files)
+            upload_info = FilestoreManager.get_signed_upload_urls(fs["id"], files_for_upload=files)
+
+            upload_urls = upload_info.get("urls")
+            existing_blobs = upload_info.get("existing")
 
             for local_path, relative_path, size in files:
-                progress.batch_title = progress_title(local_path)
-                progress.refresh()
+                skip_upload = False
+                existing_blob_info = existing_blobs.get(relative_path)
+                if existing_blob_info:
+                    modified = os.path.getmtime(local_path)
+                    if size == existing_blob_info["size"] and modified < existing_blob_info["modified"]:
+                        skip_upload = True
+
+                if not skip_upload:
+                    progress.batch_title = progress_title(local_path)
+                    progress.refresh()
 
-                FilestoreManager.upload_to_signed_url(local_path, upload_urls[relative_path])
+                    FilestoreManager.upload_to_signed_url(local_path, upload_urls[relative_path])
 
                 done_files += 1
                 done_bytes += size
@@ -201,9 +212,7 @@ class FilestoreManagerWithoutHttp:
     @classmethod
     def get_signed_upload_urls(cls, fs_id, files_for_upload):
         paths = [p for _, p, _ in files_for_upload]  # relative paths
-        return cls.make_req(f"/api/v2/filestore/fs/{fs_id}/signed-urls/upload", post=True, data={"paths": paths}).get(
-            "urls"
-        )
+        return cls.make_req(f"/api/v2/filestore/fs/{fs_id}/signed-urls/upload", post=True, data={"paths": paths})
 
     @classmethod
     def get_download_list_with_urls(cls, fs_id):

{coiled-1.122.0 → coiled-1.124.0}/coiled/software_utils.py

@@ -55,6 +55,7 @@ subdir_datas = {}
 
 ANY_AVAILABLE = "ANY-AVAILABLE"
 AUTH_BEARER_USERNAME = "AUTH_BEARER_TOKEN"
+CONDA_TOKEN_USERNAME = "CONDA_TOKEN"
 COILED_LOCAL_PACKAGE_PREFIX = "coiled_local_"
 DEFAULT_JSON_PYPI_URL = "https://pypi.org/pypi"
 DEFAULT_PYPI_URL = "https://pypi.org/simple"
@@ -573,7 +574,9 @@ def get_mamba_auth_dict(home_dir: Path | None = None) -> dict[str, tuple[str, st
             auth_data = json.load(f)
             for domain, auth in auth_data.items():
                 auth_type = auth.get("type")
-                if auth_type == "CondaToken" or auth_type == "BearerToken":
+                if auth_type == "CondaToken":
+                    domain_auth[domain] = (CONDA_TOKEN_USERNAME, auth["token"])
+                elif auth_type == "BearerToken":
                     domain_auth[domain] = (AUTH_BEARER_USERNAME, auth["token"])
                 elif auth_type == "BasicHTTPAuthentication":
                     domain_auth[domain] = (
@@ -592,6 +595,126 @@ def get_mamba_auth(netloc: str) -> tuple[str, str] | None:
     return get_mamba_auth_dict().get(netloc, None)
 
 
+def _parse_rattler_auth_data(auth_data: dict) -> tuple[str, str] | None:
+    """Parse rattler authentication data into username/password tuple.
+
+    Handles rattler Authentication variants:
+    - {"BearerToken": "token"}
+    - {"CondaToken": "token"}
+    - {"BasicHTTP": {"username": "user", "password": "pass"}}
+
+    Returns:
+        Tuple of (username, password) or None if unknown auth type
+    """
+    if "BearerToken" in auth_data:
+        return (AUTH_BEARER_USERNAME, auth_data["BearerToken"])
+    elif "CondaToken" in auth_data:
+        return (CONDA_TOKEN_USERNAME, auth_data["CondaToken"])
+    elif "BasicHTTP" in auth_data:
+        basic_auth = auth_data["BasicHTTP"]
+        return (
+            basic_auth.get("username", ""),
+            basic_auth.get("password", ""),
+        )
+    else:
+        return None
+
+
+@functools.lru_cache
+def get_rattler_auth_dict(home_dir: Path | None = None) -> dict[str, tuple[str, str]]:
+    # RATTLER_AUTH_FILE is an env var that can override the default location
+    env_auth_file = os.environ.get("RATTLER_AUTH_FILE")
+    if env_auth_file:
+        auth_file = Path(env_auth_file)
+    else:
+        if home_dir is None:
+            home_dir = Path.home()
+        auth_file = home_dir / ".rattler" / "credentials.json"
+    domain_auth = {}
+    if auth_file.exists():
+        with auth_file.open("r") as f:
+            auth_data = json.load(f)
+            for domain, auth in auth_data.items():
+                parsed_auth = _parse_rattler_auth_data(auth)
+                if parsed_auth:
+                    domain_auth[domain] = parsed_auth
+                else:
+                    logger.debug(f"Encountered unknown rattler auth type {list(auth.keys())} for domain {domain}")
+    return domain_auth
+
+
+def get_rattler_keyring_auth(netloc: str) -> tuple[str, str] | None:
+    """Returns the Requests tuple auth for a given domain from rattler keyring storage."""
+    if not HAVE_KEYRING:
+        logger.debug("keyring not available, skipping rattler keyring auth")
+        return None
+
+    def try_keyring_auth(host: str) -> tuple[str, str] | None:
+        """Try to get auth from keyring for a specific host using rattler's storage format."""
+        try:
+            # Use the existing get_keyring_auth function with "rattler" as the URL
+            # and the host as the username to get rattler-stored credentials
+            auth_parts = get_keyring_auth("rattler", host)
+            if auth_parts:
+                username, password = auth_parts
+                if password:
+                    try:
+                        auth_data = json.loads(password)
+                        parsed_auth = _parse_rattler_auth_data(auth_data)
+                        if parsed_auth:
+                            return parsed_auth
+                    except json.JSONDecodeError:
+                        # If it's not JSON, treat it as a simple username/password
+                        return (username or host, password)
+
+        except Exception as e:
+            logger.debug(f"Error getting rattler keyring auth for {host}: {e}")
+            return None
+
+        return None
+
+    # Try exact match first
+    auth_parts = try_keyring_auth(netloc)
+    if auth_parts:
+        logger.debug(f"Found rattler keyring auth for {netloc}")
+        return auth_parts
+
+    # Try parent domain matches if exact match failed
+    # If looking for foo.example.com, try example.com (but not com)
+    parts = netloc.split(".")
+    for i in range(1, len(parts) - 1):  # Stop before single TLD
+        parent_domain = ".".join(parts[i:])
+
+        auth_parts = try_keyring_auth(parent_domain)
+        if auth_parts:
+            logger.debug(f"Found rattler keyring auth for {parent_domain} (matching {netloc})")
+            return auth_parts
+
+    logger.debug(f"No rattler keyring auth found for {netloc}")
+    return None
+
+
+def get_rattler_auth(netloc: str) -> tuple[str, str] | None:
+    """Returns the Requests tuple auth for a given domain from rattler keyring or auth file."""
+    # Try keyring first (primary storage method for rattler/pixi)
+    auth_parts = get_rattler_keyring_auth(netloc)
+    if auth_parts:
+        return auth_parts
+
+    # Fall back to file-based storage
+    # rattler allows wildcards, so we have to check for exact matches first
+    auth_parts = get_rattler_auth_dict().get(netloc, None)
+    if auth_parts:
+        return auth_parts
+
+    # then check for wildcard matches in file storage
+    for domain, auth in get_rattler_auth_dict().items():
+        if domain.startswith("*.") and netloc.endswith(domain[1:]):
+            return auth
+
+    return None
+
+
 @functools.lru_cache
 def get_conda_config() -> dict:
     """Returns the current conda config as dictionary"""
@@ -706,7 +829,7 @@ print(json.dumps(auth_parts))
                 return None
 
     if auth_type == "token":
-        username = AUTH_BEARER_USERNAME
+        username = CONDA_TOKEN_USERNAME
 
     if not username and not password:
         return None
@@ -743,6 +866,7 @@ def set_auth_for_url(url: Url | str) -> str:
         use_keyring = dask.config.get("coiled.package_sync.conda.cred_sources.keyring", True)
         use_conda_auth = dask.config.get("coiled.package_sync.conda.cred_sources.conda", True)
         use_mamba_auth = dask.config.get("coiled.package_sync.conda.cred_sources.mamba", True)
+        use_rattler_auth = dask.config.get("coiled.package_sync.conda.cred_sources.rattler", True)
 
         no_auth_url = parsed_url._replace(auth=None).url
         auth_parts = (
@@ -756,10 +880,12 @@ def set_auth_for_url(url: Url | str) -> str:
             or (get_conda_auth(no_auth_url) if use_conda_auth else None)
             # mamba could have URL stored by netloc/path or netloc
             or ((get_mamba_auth(f"{netloc}{path}") or get_mamba_auth(netloc)) if use_mamba_auth else None)
+            # rattler/pixi stores URL stored by netloc in keyring or a fallback file
+            or (get_rattler_auth(netloc) if use_rattler_auth else None)
         )
         if auth_parts is not None:
             username, password = auth_parts
-        if username == AUTH_BEARER_USERNAME:
+        if username == CONDA_TOKEN_USERNAME:
             # If the username indicates this is a token (which only happens for mamba auth)
             # the token should be embedded directly in the URL and not in the auth portion
 
@@ -775,8 +901,11 @@ def set_auth_for_url(url: Url | str) -> str:
         elif username or password:
             parsed_url = parsed_url._replace(auth=f"{username or ''}:{password or ''}")
 
-        if username and username != AUTH_BEARER_USERNAME and not password:
-            logger.info(f"No password found for {parsed_url.url}")
+        if username and username != CONDA_TOKEN_USERNAME and not password:
+            if username == AUTH_BEARER_USERNAME:
+                logger.warning(f"No bearer token found for {parsed_url.url}")
+            else:
+                logger.info(f"No password found for {parsed_url.url}")
         elif not username:
             logger.info(f"No username or password found for {parsed_url.url}")