lightning-sdk 0.1.3__py3-none-any.whl → 0.1.47__py3-none-any.whl

lightning_sdk/mmt/v1.py

@@ -0,0 +1,209 @@
+from typing import TYPE_CHECKING, Dict, Optional, Tuple, Union
+
+from lightning_sdk.api.mmt_api import MMTApiV1
+from lightning_sdk.job.v1 import _internal_status_to_external_status
+from lightning_sdk.job.work import Work
+
+if TYPE_CHECKING:
+    from lightning_sdk.machine import Machine
+    from lightning_sdk.organization import Organization
+    from lightning_sdk.status import Status
+    from lightning_sdk.studio import Studio
+    from lightning_sdk.teamspace import Teamspace
+    from lightning_sdk.user import User
+
+from lightning_sdk.mmt.base import _BaseMMT
+
+
+class _MMTV1(_BaseMMT):
+    """V1 Implementation of Multi-Machine Training."""
+
+    def __init__(
+        self,
+        name: str,
+        teamspace: Union[str, "Teamspace", None] = None,
+        org: Union[str, "Organization", None] = None,
+        user: Union[str, "User", None] = None,
+        *,
+        _fetch_job: bool = True,
+    ) -> None:
+        """Fetch already existing jobs.
+
+        Args:
+            name: the name of the job
+            teamspace: the teamspace the job is part of
+            org: the name of the organization owning the :param`teamspace` in case it is owned by an org
+            user: the name of the user owning the :param`teamspace`
+                in case it is owned directly by a user instead of an org.
+        """
+        self._job_api = MMTApiV1()
+        super().__init__(name=name, teamspace=teamspace, org=org, user=user, _fetch_job=_fetch_job)
+
+    def _submit(
+        self,
+        num_machines: int,
+        machine: "Machine",
+        command: Optional[str] = None,
+        studio: Optional["Studio"] = None,
+        image: Optional[str] = None,
+        env: Optional[Dict[str, str]] = None,
+        interruptible: bool = False,
+        cloud_account: Optional[str] = None,
+        image_credentials: Optional[str] = None,
+        cloud_account_auth: bool = False,
+        artifacts_local: Optional[str] = None,
+        artifacts_remote: Optional[str] = None,
+        entrypoint: str = "sh -c",
+    ) -> "_MMTV1":
+        """Submit a new multi-machine job to the Lightning AI platform.
+
+        Args:
+            num_machines: The number of machines to run on.
+            machine: The machine type to run the job on. One of {", ".join(_MACHINE_VALUES)}.
+            command: The command to run inside your job. Required if using a studio. Optional if using an image.
+                If not provided for images, will run the container entrypoint and default command.
+            studio: The studio env to run the job with. Mutually exclusive with image.
+            image: The docker image to run the job with. Mutually exclusive with studio.
+            env: Environment variables to set inside the job.
+            interruptible: Whether the job should run on interruptible instances. They are cheaper but can be preempted.
+            cloud_account: The cloud account to run the job on.
+                Defaults to the studio cloud account if running with studio compute env.
+                If not provided will fall back to the teamspaces default cloud account.
+            image_credentials: The credentials used to pull the image. Required if the image is private.
+                This should be the name of the respective credentials secret created on the Lightning AI platform.
+            cloud_account_auth: Whether to authenticate with the cloud account to pull the image.
+                Required if the registry is part of a cloud provider (e.g. ECR).
+            artifacts_local: The path of inside the docker container, you want to persist images from.
+                CAUTION: When setting this to "/", it will effectively erase your container.
+                Only supported for jobs with a docker image compute environment.
+            artifacts_remote: The remote storage to persist your artifacts to.
+                Should be of format <CONNECTION_TYPE>:<CONNECTION_NAME>:<PATH_WITHIN_CONNECTION>.
+                PATH_WITHIN_CONNECTION hereby is a path relative to the connection's root.
+                E.g. efs:data:some-path would result in an EFS connection named `data` and to the path `some-path`
+                within it.
+                Note that the connection needs to be added to the teamspace already in order for it to be found.
+                Only supported for jobs with a docker image compute environment.
+            entrypoint: The entrypoint of your docker container. Defaults to `sh -c` which
+                just runs the provided command in a standard shell.
+                To use the pre-defined entrypoint of the provided image, set this to an empty string.
+                Only applicable when submitting docker jobs.
+        """
+        if studio is None:
+            raise ValueError("Studio is required for submitting jobs")
+        if image is not None or image_credentials is not None or cloud_account_auth or entrypoint != "sh -c":
+            raise ValueError("Image is not supported for submitting jobs")
+
+        if artifacts_local is not None or artifacts_remote is not None:
+            raise ValueError("Specifying how to persist artifacts is not yet supported with jobs")
+
+        if env is not None:
+            raise ValueError("Environment variables are not supported for submitting jobs")
+        if command is None:
+            raise ValueError("Command is required for submitting multi-machine jobs")
+
+        _submitted = self._job_api.submit_job(
+            name=self._name,
+            num_machines=num_machines,
+            command=command,
+            studio_id=studio._studio.id,
+            teamspace_id=self._teamspace.id,
+            cloud_account=cloud_account or "",
+            machine=machine,
+            interruptible=interruptible,
+            strategy="parallel",
+        )
+
+        self._name = _submitted.name
+        self._job = _submitted
+        return self
+
+    def _update_internal_job(self) -> None:
+        try:
+            self._job = self._job_api.get_job(self._name, self.teamspace.id)
+        except ValueError as e:
+            raise ValueError(f"Job {self._name} does not exist in Teamspace {self.teamspace.name}") from e
+
+    @property
+    def machines(self) -> Tuple["Work", ...]:
+        """Returns the sub-jobs for each individual instance."""
+        works = self._job_api.list_works(self._guaranteed_job.id, self.teamspace.id)
+
+        return tuple(Work(w.id, self, self.teamspace) for w in works)
+
+    def stop(self) -> None:
+        """Stops the job."""
+        self._job_api.stop_job(self._guaranteed_job.id, self.teamspace.id)
+
+    def delete(self) -> None:
+        """Deletes the job.
+
+        Caution: This also deletes all artifacts and snapshots associated with the job.
+        """
+        self._job_api.delete_job(self._guaranteed_job.id, self.teamspace.id)
+
+    @property
+    def status(self) -> "Status":
+        """The current status of the job."""
+        try:
+            status = self._job_api.get_job_status(self._job.id, self.teamspace.id)
+            return _internal_status_to_external_status(status)
+        except Exception:
+            raise RuntimeError(
+                f"MMT {self._name} does not exist in Teamspace {self.teamspace.name}. Did you delete it?"
+            ) from None
+
+    @property
+    def artifact_path(self) -> Optional[str]:
+        """Path to the artifacts created by the job within the distributed teamspace filesystem."""
+        return f"/teamspace/jobs/{self.name}"
+
+    @property
+    def snapshot_path(self) -> Optional[str]:
+        """Path to the studio snapshot used to create the job within the distributed teamspace filesystem."""
+        return f"/teamspace/jobs/{self.name}/snapshot"
+
+    @property
+    def machine(self) -> "Machine":
+        """Returns the machine type this job is running on."""
+        return self.machines[0].machine
+
+    @property
+    def name(self) -> str:
+        """The job's name."""
+        return self._name
+
+    @property
+    def teamspace(self) -> "Teamspace":
+        """The teamspace the job is part of."""
+        return self._teamspace
+
+    @property
+    def link(self) -> str:
+        return f"https://lightning.ai/{self.teamspace.owner.name}/{self.teamspace.name}/studios/{self._job_api.get_studio_name(self._guaranteed_job)}/app?app_id=mmt&app_tab=Runs&job_name={self.name}"
+
+    @property
+    def image(self) -> Optional[str]:
+        """The image used to submit the job."""
+        # mmtv1 don't support images, so return None here
+        return None
+
+    @property
+    def studio(self) -> Optional["Studio"]:
+        """The studio used to submit the job."""
+        from lightning_sdk.studio import Studio
+
+        studio_name = self._job_api.get_studio_name(self._guaranteed_job)
+        return Studio(studio_name, teamspace=self.teamspace)
+
+    @property
+    def command(self) -> str:
+        """The command the job is running."""
+        return self._job_api.get_command(self._guaranteed_job)
+
+    # the following and functions are solely to make the Work class function
+    @property
+    def _id(self) -> str:
+        return self._guaranteed_job.id
+
+    def _name_filter(self, name: str) -> str:
+        return name.replace("root.", "")

lightning_sdk/mmt/v2.py

@@ -0,0 +1,221 @@
+from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union
+
+from lightning_sdk.api.mmt_api import MMTApiV2
+
+if TYPE_CHECKING:
+    from lightning_sdk.job.job import Job
+    from lightning_sdk.machine import Machine
+    from lightning_sdk.organization import Organization
+    from lightning_sdk.status import Status
+    from lightning_sdk.studio import Studio
+    from lightning_sdk.teamspace import Teamspace
+    from lightning_sdk.user import User
+
+from lightning_sdk.mmt.base import _BaseMMT
+
+
+class _MMTV2(_BaseMMT):
+    """New implementation of Multi-Machine Training."""
+
+    def __init__(
+        self,
+        name: str,
+        teamspace: Union[str, "Teamspace", None] = None,
+        org: Union[str, "Organization", None] = None,
+        user: Union[str, "User", None] = None,
+        *,
+        _fetch_job: bool = True,
+    ) -> None:
+        """Fetch already existing jobs.
+
+        Args:
+            name: the name of the job
+            teamspace: the teamspace the job is part of
+            org: the name of the organization owning the :param`teamspace` in case it is owned by an org
+            user: the name of the user owning the :param`teamspace`
+                in case it is owned directly by a user instead of an org.
+        """
+        self._job_api = MMTApiV2()
+        super().__init__(name=name, teamspace=teamspace, org=org, user=user, _fetch_job=_fetch_job)
+
+    def _submit(
+        self,
+        num_machines: int,
+        machine: "Machine",
+        command: Optional[str] = None,
+        studio: Optional["Studio"] = None,
+        image: Optional[str] = None,
+        env: Optional[Dict[str, str]] = None,
+        interruptible: bool = False,
+        cloud_account: Optional[str] = None,
+        image_credentials: Optional[str] = None,
+        cloud_account_auth: bool = False,
+        artifacts_local: Optional[str] = None,
+        artifacts_remote: Optional[str] = None,
+        entrypoint: str = "sh -c",
+    ) -> "_MMTV2":
+        """Submit a new multi-machine job to the Lightning AI platform.
+
+        Args:
+            num_machines: The number of machines to run on.
+            machine: The machine type to run the job on. One of {", ".join(_MACHINE_VALUES)}.
+            command: The command to run inside your job. Required if using a studio. Optional if using an image.
+                If not provided for images, will run the container entrypoint and default command.
+            studio: The studio env to run the job with. Mutually exclusive with image.
+            image: The docker image to run the job with. Mutually exclusive with studio.
+            env: Environment variables to set inside the job.
+            interruptible: Whether the job should run on interruptible instances. They are cheaper but can be preempted.
+            cloud_account: The cloud account to run the job on.
+                Defaults to the studio cloud account if running with studio compute env.
+                If not provided will fall back to the teamspaces default cloud account.
+            image_credentials: The credentials used to pull the image. Required if the image is private.
+                This should be the name of the respective credentials secret created on the Lightning AI platform.
+            cloud_account_auth: Whether to authenticate with the cloud account to pull the image.
+                Required if the registry is part of a cloud provider (e.g. ECR).
+            artifacts_local: The path of inside the docker container, you want to persist images from.
+                CAUTION: When setting this to "/", it will effectively erase your container.
+                Only supported for jobs with a docker image compute environment.
+            artifacts_remote: The remote storage to persist your artifacts to.
+                Should be of format <CONNECTION_TYPE>:<CONNECTION_NAME>:<PATH_WITHIN_CONNECTION>.
+                PATH_WITHIN_CONNECTION hereby is a path relative to the connection's root.
+                E.g. efs:data:some-path would result in an EFS connection named `data` and to the path `some-path`
+                within it.
+                Note that the connection needs to be added to the teamspace already in order for it to be found.
+                Only supported for jobs with a docker image compute environment.
+            entrypoint: The entrypoint of your docker container. Defaults to `sh -c` which
+                just runs the provided command in a standard shell.
+                To use the pre-defined entrypoint of the provided image, set this to an empty string.
+                Only applicable when submitting docker jobs.
+        """
+        # Command is required if Studio is provided to know what to run
+        # Image is mutually exclusive with Studio
+        # Command is optional for Image
+        # Either image or studio must be provided
+        if studio is not None:
+            studio_id = studio._studio.id
+            if image is not None:
+                raise ValueError(
+                    "image and studio are mutually exclusive as both define the environment to run the job in"
+                )
+            if command is None:
+                raise ValueError("command is required when using a studio")
+        else:
+            studio_id = None
+            if image is None:
+                raise ValueError("either image or studio must be provided")
+        submitted = self._job_api.submit_job(
+            name=self.name,
+            num_machines=num_machines,
+            command=command,
+            cloud_account=cloud_account,
+            teamspace_id=self._teamspace.id,
+            studio_id=studio_id,
+            image=image,
+            machine=machine,
+            interruptible=interruptible,
+            env=env,
+            image_credentials=image_credentials,
+            cloud_account_auth=cloud_account_auth,
+            artifacts_local=artifacts_local,
+            artifacts_remote=artifacts_remote,
+            entrypoint=entrypoint,
+        )
+        self._job = submitted
+        self._name = submitted.name
+        return self
+
+    @property
+    def machines(self) -> Tuple["Job", ...]:
+        """Returns the sub-jobs for each individual instance."""
+        from lightning_sdk.job import Job
+
+        return tuple(
+            Job(name=j.name, teamspace=self.teamspace)
+            for j in self._job_api.list_mmt_subjobs(self._guaranteed_job.id, self.teamspace.id)
+        )
+
+    def stop(self) -> None:
+        """Stops the job."""
+        self._job_api.stop_job(job_id=self._guaranteed_job.id, teamspace_id=self._teamspace.id)
+
+    def delete(self) -> None:
+        """Deletes the job.
+
+        Caution: This also deletes all artifacts and snapshots associated with the job.
+        """
+        self._job_api.delete_job(
+            job_id=self._guaranteed_job.id,
+            teamspace_id=self._teamspace.id,
+        )
+
+    @property
+    def _latest_job(self) -> Any:
+        """Guarantees to fetch the latest version of a job before returning it."""
+        self._update_internal_job()
+        return self._job
+
+    @property
+    def status(self) -> "Status":
+        """The current status of the job."""
+        return self._job_api._job_state_to_external(self._latest_job.state)
+
+    @property
+    def artifact_path(self) -> Optional[str]:
+        """Path to the artifacts created by the job within the distributed teamspace filesystem."""
+        # TODO: Since grouping for those is not done yet on the BE, we cannot yet have a unified link here
+        raise NotImplementedError
+
+    @property
+    def snapshot_path(self) -> Optional[str]:
+        """Path to the studio snapshot used to create the job within the distributed teamspace filesystem."""
+        # TODO: Since grouping for those is not done yet on the BE, we cannot yet have a unified link here
+        raise NotImplementedError
+
+    @property
+    def machine(self) -> "Machine":
+        """Returns the machine type this job is running on."""
+        return self._job_api._get_job_machine_from_spec(self._guaranteed_job.spec)
+
+    def _update_internal_job(self) -> None:
+        if getattr(self, "_job", None) is None:
+            self._job = self._job_api.get_job_by_name(name=self._name, teamspace_id=self._teamspace.id)
+            return
+
+        self._job = self._job_api.get_job(job_id=self._job.id, teamspace_id=self._teamspace.id)
+
+    @property
+    def name(self) -> str:
+        """The job's name."""
+        return self._name
+
+    @property
+    def teamspace(self) -> "Teamspace":
+        """The teamspace the job is part of."""
+        return self._teamspace
+
+    @property
+    def link(self) -> str:
+        # TODO: Since we don't have a UI for this yet, we can't have a link
+        raise NotImplementedError
+
+    @property
+    def image(self) -> Optional[str]:
+        """The image used to submit the job."""
+        return self._job_api.get_image_name(self._guaranteed_job)
+
+    @property
+    def studio(self) -> Optional["Studio"]:
+        """The studio used to submit the job."""
+        from lightning_sdk.studio import Studio
+
+        studio_name = self._job_api.get_studio_name(self._guaranteed_job)
+
+        # if job was submitted with image, studio will be None
+        if not studio_name:
+            return None
+        return Studio(studio_name, teamspace=self.teamspace)
+
+    @property
+    def command(self) -> str:
+        """The command the job is running."""
+        return self._job_api.get_command(self._guaranteed_job)

lightning_sdk/models.py

@@ -0,0 +1,153 @@
+import os
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
+
+from lightning_sdk.api import OrgApi, TeamspaceApi, UserApi
+from lightning_sdk.lightning_cloud.openapi.models import V1Membership, V1OwnerType
+from lightning_sdk.lightning_cloud.openapi.rest import ApiException
+from lightning_sdk.user import User
+from lightning_sdk.utils.resolve import _get_authed_user
+
+if TYPE_CHECKING:
+    from lightning_sdk.teamspace import Teamspace
+
+
+# TODO: Maybe just have a `Model` object?
+@dataclass
+class UploadedModelInfo:
+    name: str
+    version: str
+    teamspace: str
+    cloud_account: str
+
+
+def _get_teamspace_and_path(
+    ts: V1Membership, org_api: OrgApi, user_api: UserApi, authed_user: User
+) -> Tuple[str, Dict[str, Any]]:
+    if ts.owner_type == V1OwnerType.ORGANIZATION:
+        org = org_api._get_org_by_id(ts.owner_id)
+        return f"{org.name}/{ts.name}", {"name": ts.name, "org": org.name}
+
+    if ts.owner_type == V1OwnerType.USER and ts.owner_id != authed_user.id:
+        user = user_api._get_user_by_id(ts.owner_id)  # todo: check also the name
+        return f"{user.username}/{ts.name}", {"name": ts.name, "user": User(name=user.username)}
+
+    if ts.owner_type == V1OwnerType.USER:
+        return f"{authed_user.name}/{ts.name}", {"name": ts.name, "user": authed_user}
+
+    raise RuntimeError(f"Unknown organization type {ts.owner_type}")
+
+
+def _list_teamspaces() -> List[str]:
+    org_api = OrgApi()
+    user_api = UserApi()
+    authed_user = _get_authed_user()
+
+    return [
+        _get_teamspace_and_path(ts, org_api, user_api, authed_user)[0]
+        for ts in user_api._get_all_teamspace_memberships("")
+    ]
+
+
+def _get_teamspace(name: str, organization: str) -> "Teamspace":
+    """Get a Teamspace object from the SDK."""
+    from lightning_sdk.teamspace import Teamspace
+
+    org_api = OrgApi()
+    user_api = UserApi()
+    authed_user = _get_authed_user()
+
+    requested_teamspace = f"{organization}/{name}".lower()
+
+    for ts in user_api._get_all_teamspace_memberships(""):
+        if ts.name != name:
+            continue
+
+        teamspace_path, teamspace = _get_teamspace_and_path(ts, org_api, user_api, authed_user)
+        if requested_teamspace == teamspace_path:
+            return Teamspace(**teamspace)
+
+    options = f"{os.linesep}\t".join(_list_teamspaces())
+    raise RuntimeError(f"Teamspace `{requested_teamspace}` not found. Available teamspaces: {os.linesep}\t{options}")
+
+
+def _parse_model_name_and_version(name: str) -> Tuple[str, str, str, str]:
+    """Parse the name argument into its components."""
+    try:
+        org_name, teamspace_name, model_name = name.split("/")
+        parts = model_name.split(":")
+        if len(parts) == 1:
+            return org_name, teamspace_name, parts[0], "latest"
+        if len(parts) == 2:
+            return org_name, teamspace_name, parts[0], parts[1]
+        # The rest of the validation for name and version happens in the backend
+        raise ValueError(
+            "Model version is expected to be in the format `entity/modelname:version` separated by a"
+            f" single colon, but got: {name}"
+        )
+    except ValueError as err:
+        raise ValueError(
+            f"Model name must be in the format 'organization/teamspace/model' but you provided '{name}'."
+        ) from err
+
+
+def download_model(
+    name: str,
+    download_dir: Union[Path, str] = ".",
+    progress_bar: bool = True,
+) -> List[str]:
+    """Download a Model.
+
+    Args:
+        name: The name of the Model you want to download.
+        This should have the format <ORGANIZATION-NAME>/<TEAMSPACE-NAME>/<MODEL-NAME>.
+        download_dir: The directory where the Model should be downloaded.
+        progress_bar: Whether to show a progress bar when downloading.
+    """
+    teamspace_owner_name, teamspace_name, model_name, version = _parse_model_name_and_version(name)
+
+    download_dir = Path(download_dir)
+
+    api = TeamspaceApi()
+
+    try:
+        return api.download_model_files(
+            name=model_name,
+            version=version,
+            download_dir=download_dir,
+            teamspace_name=teamspace_name,
+            teamspace_owner_name=teamspace_owner_name,
+            progress_bar=progress_bar,
+        )
+    except ApiException as e:
+        # if we get an error, check if the teamspace actually exists (and print the list)
+        # TODO: ideally this would match a specific error about teamspace not being found
+        _ = _get_teamspace(name=teamspace_name, organization=teamspace_owner_name)
+        raise e
+
+
+def upload_model(
+    name: str,
+    path: Union[Path, str] = ".",
+    cloud_account: Optional[str] = None,
+    progress_bar: bool = True,
+) -> UploadedModelInfo:
+    """Upload a Model.
+
+    Args:
+        name: The name of the Model you want to upload.
+            This should have the format <ORGANIZATION-NAME>/<TEAMSPACE-NAME>/<MODEL-NAME>.
+        path: The path to the file or directory you want to upload. Defaults to the current directory.
+        cloud_account: The name of the cloud account to store the Model in.
+            If not provided, the default cloud account for the Teamspace will be used.
+        progress_bar: Whether to show a progress bar for the upload.
+    """
+    org_name, teamspace_name, model_name, _ = _parse_model_name_and_version(name)
+    teamspace = _get_teamspace(name=teamspace_name, organization=org_name)
+    return teamspace.upload_model(
+        path=path,
+        name=model_name,
+        cloud_account=cloud_account,
+        progress_bar=progress_bar,
+    )

lightning_sdk/organization.py

@@ -2,7 +2,7 @@ from typing import Optional
 
 from lightning_sdk.api import OrgApi
 from lightning_sdk.owner import Owner
-from lightning_sdk.utils import _resolve_org_name
+from lightning_sdk.utils.resolve import _resolve_org_name
 
 
 class Organization(Owner):