coiled 1.128.3.dev1__tar.gz → 1.130.2.dev11__tar.gz

{coiled-1.128.3.dev1 → coiled-1.130.2.dev11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: coiled
-Version: 1.128.3.dev1
+Version: 1.130.2.dev11
 Summary: Python client for coiled.io dask clusters
 Project-URL: Homepage, https://coiled.io
 Maintainer-email: Coiled <info@coiled.io>

{coiled-1.128.3.dev1 → coiled-1.130.2.dev11}/coiled/capture_environment.py

@@ -1,6 +1,4 @@
-import asyncio
 import contextlib
-import logging
 import platform
 import sys
 import typing
@@ -16,9 +14,12 @@ from typing_extensions import Literal
 from coiled.context import track_context
 from coiled.scan import scan_prefix
 from coiled.software_utils import (
+    ANY_AVAILABLE,
+    PYTHON_VERSION,
     check_pip_happy,
     create_wheels_for_local_python,
     create_wheels_for_packages,
+    get_lockfile,
     partition_ignored_packages,
     partition_local_packages,
     partition_local_python_code_packages,
@@ -36,10 +37,6 @@ from coiled.v2.core import CloudV2
 from coiled.v2.widgets.rich import CONSOLE_WIDTH, print_rich_package_table
 from coiled.v2.widgets.util import simple_progress, use_rich_widget
 
-PYTHON_VERSION = platform.python_version_tuple()
-ANY_AVAILABLE = "ANY-AVAILABLE"
-
-
 logger = getLogger("coiled.package_sync")
 
 
@@ -69,49 +66,53 @@ async def approximate_packages(
     architecture: ArchitectureTypesEnum = ArchitectureTypesEnum.X86_64,
     pip_check_errors: Optional[Dict[str, List[str]]] = None,
     gpu_enabled: bool = False,
+    use_uv_installer: bool = True,
+    lockfile_path: Optional[Path] = None,
 ) -> typing.List[ResolvedPackageInfo]:
     user_conda_installed_python = next((p for p in packages if p["name"] == "python"), None)
-    user_conda_installed_pip = next(
-        (i for i, p in enumerate(packages) if p["name"] == "pip" and p["source"] == "conda"),
-        None,
-    )
-    if not user_conda_installed_pip:
-        # This means pip was installed by pip, or the system
-        # package manager
-        # Insert a conda version of pip to be installed first, it will
-        # then be used to install the users version of pip
-        pip = next(
-            (p for p in packages if p["name"] == "pip" and p["source"] == "pip"),
+    # Only add pip if we need it
+    if not use_uv_installer:
+        user_conda_installed_pip = next(
+            (i for i, p in enumerate(packages) if p["name"] == "pip" and p["source"] == "conda"),
             None,
         )
-        if not pip:
-            # insert a modern version and hope it does not introduce conflicts
-            packages.append({
-                "name": "pip",
-                "path": None,
-                "source": "conda",
-                "channel_url": "https://conda.anaconda.org/conda-forge/",
-                "channel": "conda-forge",
-                "subdir": "noarch",
-                "conda_name": "pip",
-                "version": "22.3.1",
-                "wheel_target": None,
-                "requested": False,
-            })
-        else:
-            # insert the users pip version and hope it exists on conda-forge
-            packages.append({
-                "name": "pip",
-                "path": None,
-                "source": "conda",
-                "channel_url": "https://conda.anaconda.org/conda-forge/",
-                "channel": "conda-forge",
-                "subdir": "noarch",
-                "conda_name": "pip",
-                "version": pip["version"],
-                "wheel_target": None,
-                "requested": True,
-            })
+        if not user_conda_installed_pip:
+            # This means pip was installed by pip, or the system
+            # package manager
+            # Insert a conda version of pip to be installed first, it will
+            # then be used to install the users version of pip
+            pip = next(
+                (p for p in packages if p["name"] == "pip" and p["source"] == "pip"),
+                None,
+            )
+            if not pip:
+                # insert a modern version and hope it does not introduce conflicts
+                packages.append({
+                    "name": "pip",
+                    "path": None,
+                    "source": "conda",
+                    "channel_url": "https://conda.anaconda.org/conda-forge/",
+                    "channel": "conda-forge",
+                    "subdir": "noarch",
+                    "conda_name": "pip",
+                    "version": "22.3.1",
+                    "wheel_target": None,
+                    "requested": False,
+                })
+            else:
+                # insert the users pip version and hope it exists on conda-forge
+                packages.append({
+                    "name": "pip",
+                    "path": None,
+                    "source": "conda",
+                    "channel_url": "https://conda.anaconda.org/conda-forge/",
+                    "channel": "conda-forge",
+                    "subdir": "noarch",
+                    "conda_name": "pip",
+                    "version": pip["version"],
+                    "wheel_target": None,
+                    "requested": True,
+                })
     coiled_selected_python = None
     if not user_conda_installed_python:
         # insert a special python package
@@ -162,6 +163,8 @@ async def approximate_packages(
             architecture=architecture,
             pip_check_errors=pip_check_errors,
             gpu_enabled=gpu_enabled,
+            lockfile_name=lockfile_path.name if lockfile_path else None,
+            lockfile_content=lockfile_path.read_text() if lockfile_path else None,
         )
     finalized_packages: typing.List[ResolvedPackageInfo] = []
     finalized_packages.extend(await create_wheels_for_local_python(local_python_code, progress=progress))
@@ -208,6 +211,7 @@ async def create_environment_approximation(
     progress: Optional[Progress] = None,
     architecture: ArchitectureTypesEnum = ArchitectureTypesEnum.X86_64,
     gpu_enabled: bool = False,
+    use_uv_installer: bool = True,
 ) -> typing.List[ResolvedPackageInfo]:
     packages = await scan_prefix(progress=progress)
     pip_check_errors = await check_pip_happy(progress)
@@ -237,6 +241,8 @@ async def create_environment_approximation(
         architecture=architecture,
         pip_check_errors=pip_check_errors,
         gpu_enabled=gpu_enabled,
+        use_uv_installer=use_uv_installer,
+        lockfile_path=get_lockfile(),
     )
     return result
 
@@ -259,7 +265,7 @@ async def scan_and_create(
 ):
     use_widget = force_rich_widget or (show_widget and use_rich_widget())
 
-    local_env_name = Path(sys.prefix).name
+    local_env_name = str(get_lockfile() or Path(sys.prefix).name)
     if use_widget:
         progress = Progress(TextColumn("[progress.description]{task.description}"), BarColumn(), TimeElapsedColumn())
         live = Live(Panel(progress, title=f"[green]Package Sync for {local_env_name}", width=CONSOLE_WIDTH))
@@ -268,6 +274,9 @@ async def scan_and_create(
         progress = None
 
     with live:
+        # We do this even with lockfiles because some early checks happen
+        # on this endpoint to prevent people getting delayed quota errors
+        # TODO: Add a lighter weight endpoint that does just these checks
         with simple_progress("Fetching latest package priorities", progress):
             logger.info(f"Resolving your local {local_env_name} Python environment...")
             async with (
@@ -306,6 +315,7 @@ async def scan_and_create(
                 architecture=architecture,
                 gpu_enabled=gpu_enabled,
                 conda_extras=package_sync_conda_extras,
+                use_uv_installer=use_uv_installer,
             )
 
         if not package_sync_only:
@@ -367,6 +377,7 @@ async def scan_and_create(
                 # default region in declarative service create_software_environment
                 region_name=region_name,
                 use_uv_installer=use_uv_installer,
+                lockfile_path=get_lockfile(),
             )
     if use_widget:
         print_rich_package_table(packages_with_notes, packages_with_errors)
@@ -427,39 +438,3 @@ If you use pip, venv, uv, pixi, etc. create a new environment and then:
 
 See https://docs.coiled.io/user_guide/software/package_sync_best_practices.html
 for more best practices. If that doesn't solve your issue, please contact support@coiled.io.""")
-
-
-if __name__ == "__main__":
-    from logging import basicConfig
-
-    basicConfig(level=logging.INFO)
-
-    from rich.console import Console
-    from rich.table import Table
-
-    async def run():
-        async with CloudV2(asynchronous=True) as cloud:
-            return await create_environment_approximation(
-                cloud=cloud,
-                priorities={
-                    ("dask", "conda"): PackageLevelEnum.CRITICAL,
-                    ("twisted", "conda"): PackageLevelEnum.IGNORE,
-                    ("graphviz", "conda"): PackageLevelEnum.LOOSE,
-                    ("icu", "conda"): PackageLevelEnum.LOOSE,
-                },
-            )
-
-    result = asyncio.run(run())
-
-    table = Table(title="Packages")
-    keys = ("name", "source", "include", "client_version", "specifier", "error", "note")
-
-    for key in keys:
-        table.add_column(key)
-
-    for pkg in result:
-        row_values = [str(pkg.get(key, "")) for key in keys]
-        table.add_row(*row_values)
-    console = Console()
-    console.print(table)
-    console.print(table)

{coiled-1.128.3.dev1 → coiled-1.130.2.dev11}/coiled/cli/hello/hello.py

@@ -295,7 +295,7 @@ Choose any computation you'd like to run:
 Yee-haw you've done all my examples 🎉  
 Now you can:
 - Try Coiled in your own use case
-- [Talk to us](https://calendly.com/d/cmph-386-cjt/coiled-help-getting-started)
+- [Ask us questions](mailto:support@coiled.io)
 - Explore the [docs](https://docs.coiled.io?utm_source=coiled-hello&utm_medium=finished) to see all the other things Coiled can do
 """),  # noqa
                     border_style="green",

{coiled-1.128.3.dev1 → coiled-1.130.2.dev11}/coiled/cli/setup/azure.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import datetime
 import json
 import os
@@ -6,7 +8,6 @@ import shutil
 import subprocess
 import sys
 import time
-from typing import Optional
 
 import click
 import httpx
@@ -158,6 +159,12 @@ coiled curl -X POST "${SETUP_ENDPOINT}" --json --data "{\\"credentials\\": {\\"t
     is_flag=True,
     help="If there's existing enterprise application, add new secret rather than replacing any existing ones.",
 )
+@click.option(
+    "--update-role-definitions",
+    default=False,
+    is_flag=True,
+    help="If Role Definitions already exist, then update them; default is to leave them unchanged if they exist.",
+)
 @click.option(
     "--save-script",
     is_flag=True,
@@ -175,8 +182,22 @@ coiled curl -X POST "${SETUP_ENDPOINT}" --json --data "{\\"credentials\\": {\\"t
         "service principal for Coiled to use."
     ),
 )
+@click.option(
+    "--refresh-for-app-id", default=None, help="Refresh the secret key used by Coiled for specified Application ID."
+)
 @click.command(context_settings=CONTEXT_SETTINGS)
-def azure_setup(subscription, resource_group, region, account, iam_user, keep_existing_access, save_script, ship_token):
+def azure_setup(
+    subscription,
+    resource_group,
+    region,
+    account,
+    iam_user,
+    keep_existing_access,
+    update_role_definitions,
+    save_script,
+    ship_token,
+    refresh_for_app_id,
+):
     print(
         "Coiled on Azure is currently in [bold]public beta[/bold], "
         "please contact [link]support@coiled.io[/link] if you have any questions or problems."
@@ -264,6 +285,17 @@ def azure_setup(subscription, resource_group, region, account, iam_user, keep_ex
         f"with [green]{region}[/green] as the default region\n"
     )
 
+    if refresh_for_app_id:
+        refresh_app_creds(
+            app_id=refresh_for_app_id,
+            coiled_account=coiled_account,
+            sub_id=sub_id,
+            rg_name=rg_name,
+            region=region,
+            keep_existing_keys=True,
+        )
+        return
+
     if ship_token:
         enable_providers(creds, sub_id)
         ship_token_creds(
@@ -289,7 +321,15 @@ def azure_setup(subscription, resource_group, region, account, iam_user, keep_ex
         app_name = iam_user or f"coiled-{coiled_account}-app"
         try:
             if not setup_with_service_principal(
-                creds, app_name, sub_id, rg_name, rg_id, coiled_account, region, keep_existing_keys=keep_existing_access
+                creds,
+                app_name,
+                sub_id,
+                rg_name,
+                rg_id,
+                coiled_account,
+                region,
+                keep_existing_keys=keep_existing_access,
+                update_role_definitions=update_role_definitions,
             ):
                 coiled.add_interaction(action="CoiledSetup", success=False)
         except Exception as e:
@@ -301,7 +341,15 @@ def azure_setup(subscription, resource_group, region, account, iam_user, keep_ex
 
 
 def setup_with_service_principal(
-    creds, app_name, sub_id, rg_name, rg_id, coiled_account, region, keep_existing_keys: bool = False
+    creds,
+    app_name,
+    sub_id,
+    rg_name,
+    rg_id,
+    coiled_account,
+    region,
+    keep_existing_keys: bool = False,
+    update_role_definitions: bool = False,
 ):
     prompt = f"Create [green]{app_name}[/green] service principal and grant Coiled access to your Azure subscription?"
     if not Confirm.ask(prompt, default=True):
@@ -341,12 +389,16 @@ def setup_with_service_principal(
 
         az_cli_wrapper(
             f"az role definition create --role-definition @{role_def_path}",
-            command_if_exists=f"az role definition update --role-definition @{role_def_path}",
+            command_if_exists=f"az role definition update --role-definition @{role_def_path}"
+            if update_role_definitions
+            else None,
         )
         print(f"  [bright_black]Creating/updating role definition {LOG_ROLE_NAME}...")
         az_cli_wrapper(
             f"az role definition create --role-definition @{log_role_def_path}",
-            command_if_exists=f"az role definition update --role-definition @{log_role_def_path}",
+            command_if_exists=f"az role definition update --role-definition @{log_role_def_path}"
+            if update_role_definitions
+            else None,
         )
 
     print(f"  [bright_black]Assigning '{RG_ROLE_NAME}' role to service principal on '{rg_name}' resource group...")
@@ -378,6 +430,31 @@ def setup_with_service_principal(
     return True
 
 
+def refresh_app_creds(app_id, keep_existing_keys, coiled_account, sub_id, rg_name, region):
+    print(f"  [bright_black]Resetting/retrieving credentials for {app_id}...")
+    cred_reset_opts = "--append" if keep_existing_keys else ""
+    app_creds_json = az_cli_wrapper(f"az ad app credential reset --id {app_id} {cred_reset_opts}")
+    app_creds = json.loads(app_creds_json)
+
+    creds_to_submit = {
+        "tenant_id": app_creds["tenant"],
+        "client_id": app_creds["appId"],
+        "client_secret": app_creds["password"],
+    }
+
+    print("Sending Azure credentials to Coiled... ", end="")
+    submit_azure_credentials(
+        coiled_account=coiled_account,
+        sub_id=sub_id,
+        rg_name=rg_name,
+        region=region,
+        creds_to_submit=creds_to_submit,
+    )
+    print(f"Azure credentials have been updated for {coiled_account} using Azure app {app_id} and {rg_name}!")
+
+    coiled.add_interaction(action="RefreshCloudCredentials", success=True)
+
+
 def submit_azure_credentials(coiled_account, sub_id, rg_name, region, creds_to_submit, check_after: bool = False):
     with coiled.Cloud(account=coiled_account) as cloud:
         setup_endpoint = f"/api/v2/cloud-credentials/{coiled_account}/azure"
@@ -399,13 +476,13 @@ def strip_output(output: str) -> str:
     return output.strip(' \n"')
 
 
-def get_cli_path() -> Optional[str]:
+def get_cli_path() -> str | None:
     return shutil.which("az")
 
 
 def az_cli_wrapper(
     command: str,
-    command_if_exists: str = "",
+    command_if_exists: str | None = "",
     show_stdout: bool = False,
     interactive: bool = False,
 ):

{coiled-1.128.3.dev1 → coiled-1.130.2.dev11}/coiled/context.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
-import asyncio
 import functools
+import inspect
 import random
 import string
 from contextlib import contextmanager, nullcontext
@@ -103,7 +103,7 @@ def get_trace_context(func: Union[SyncFuncType, AsyncFuncType]):
 
 
 def track_context(func: F) -> F:
-    if asyncio.iscoroutinefunction(func):
+    if inspect.iscoroutinefunction(func):
 
         @functools.wraps(func)
         async def async_wrapper(*args: Any, **kwargs: Any) -> Any:

{coiled-1.128.3.dev1 → coiled-1.130.2.dev11}/coiled/core.py

@@ -924,6 +924,7 @@ class Cloud(Generic[IsAsynchronous]):
         include_local_code: bool = False,
         ignore_local_packages: List[str] | None = None,
         use_uv_installer: bool = True,
+        lockfile_path: Union[str, pathlib.Path, None] = None,
     ) -> SoftwareEnvironmentAlias | None:
         if name is None and conda is not None and isinstance(conda, dict) and "name" in conda:
             name = conda["name"]
@@ -943,8 +944,20 @@ class Cloud(Generic[IsAsynchronous]):
             raise TypeError("The build backend does not support specifying both packages and a container")
         if container and include_local_code:
             raise TypeError("The build backend does not support including local code when using a container")
-        if conda or pip:
-            senv = await create_env_spec(conda=conda, pip=pip)
+        if lockfile_path:
+            if pip or conda:
+                raise TypeError("The build backend does not support specifying both a lockfile and packages")
+            lockfile_path = pathlib.Path(lockfile_path)
+            if not lockfile_path.exists():
+                raise FileNotFoundError(f"Lockfile not found: {lockfile_path}")
+            if not lockfile_path.name.endswith(("uv.lock", "pylock.toml", "conda-lock.yml")):
+                logger.warning(
+                    "The specified lockfile does not appear to be generated by a supported tool "
+                    "(uv, pip, conda-lock). Proceeding anyway."
+                )
+
+        if conda or pip or lockfile_path:
+            senv = await create_env_spec(conda=conda, pip=pip, lockfile_path=lockfile_path)
             if include_local_code:
                 prefix = await scan_prefix()
                 packages, _ = partition_ignored_packages(
@@ -1030,6 +1043,8 @@ class Cloud(Generic[IsAsynchronous]):
         architecture: ArchitectureTypesEnum,
         pip_check_errors: Dict[str, List[str]] | None = None,
         gpu_enabled: bool = False,
+        lockfile_name: str | None = None,
+        lockfile_content: str | None = None,
     ) -> List[ApproximatePackageResult]:
         response = await self._do_request(
             "POST",
@@ -1046,6 +1061,8 @@ class Cloud(Generic[IsAsynchronous]):
                 "index_urls": get_index_urls(),
                 "pip_check_errors": pip_check_errors,
                 "gpu_enabled": gpu_enabled,
+                "lockfile_name": lockfile_name,
+                "lockfile_content": lockfile_content,
             },
         )
         if response.status >= 400:
@@ -1133,6 +1150,8 @@ class Cloud(Generic[IsAsynchronous]):
             "architecture": architecture,
             "region_name": region_name,
             "enable_experimental_installer": use_uv_installer,
+            "lockfile_content": senv.get("lockfile_content") if senv else None,
+            "lockfile_name": senv.get("lockfile_name") if senv else None,
         }
         if senv:
             payload["packages"] = senv["packages"]

{coiled-1.128.3.dev1 → coiled-1.130.2.dev11}/coiled/filestore.py

@@ -422,7 +422,7 @@ class FilestoreManagerWithoutHttp:
                             for chunk in response.iter_bytes(chunk_size=8192):
                                 f.write(chunk)
                     return  # Success, exit function
-                except (httpx.RemoteProtocolError, httpx.ReadTimeout, httpx.ConnectError) as e:
+                except (httpx.RemoteProtocolError, httpx.ReadTimeout, httpx.ConnectError, httpx.HTTPStatusError) as e:
                     if attempt < max_retries - 1:
                         wait_time = 2**attempt  # Exponential backoff: 1s, 2s, 4s
                         if verbose: