lightning-sdk 0.2.22__py3-none-any.whl → 0.2.23__py3-none-any.whl

lightning_sdk/lightning_cloud/openapi/models/v1_volume.py

@@ -43,21 +43,20 @@ class V1Volume(object):
     swagger_types = {
         'attached_at': 'datetime',
         'availability_zone': 'str',
-        'cluster_id': 'str',
-        'created_at': 'datetime',
         'detached_at': 'datetime',
         'encrypted': 'bool',
-        'id': 'str',
         'iops': 'str',
+        'metadata': 'V1Metadata',
         'path': 'str',
-        'project_id': 'str',
         'provider': 'str',
         'provider_id': 'str',
         'region': 'str',
         'resource_id': 'str',
         'resource_type': 'str',
+        'retention_period': 'str',
         'server_id': 'str',
         'size_gb': 'str',
+        'state': 'V1VolumeState',
         'throughput': 'str',
         'type': 'str',
         'updated_at': 'datetime',
@@ -67,46 +66,44 @@ class V1Volume(object):
     attribute_map = {
         'attached_at': 'attachedAt',
         'availability_zone': 'availabilityZone',
-        'cluster_id': 'clusterId',
-        'created_at': 'createdAt',
         'detached_at': 'detachedAt',
         'encrypted': 'encrypted',
-        'id': 'id',
         'iops': 'iops',
+        'metadata': 'metadata',
         'path': 'path',
-        'project_id': 'projectId',
         'provider': 'provider',
         'provider_id': 'providerId',
         'region': 'region',
         'resource_id': 'resourceId',
         'resource_type': 'resourceType',
+        'retention_period': 'retentionPeriod',
         'server_id': 'serverId',
         'size_gb': 'sizeGb',
+        'state': 'state',
         'throughput': 'throughput',
         'type': 'type',
         'updated_at': 'updatedAt',
         'user_id': 'userId'
     }
 
-    def __init__(self, attached_at: 'datetime' =None, availability_zone: 'str' =None, cluster_id: 'str' =None, created_at: 'datetime' =None, detached_at: 'datetime' =None, encrypted: 'bool' =None, id: 'str' =None, iops: 'str' =None, path: 'str' =None, project_id: 'str' =None, provider: 'str' =None, provider_id: 'str' =None, region: 'str' =None, resource_id: 'str' =None, resource_type: 'str' =None, server_id: 'str' =None, size_gb: 'str' =None, throughput: 'str' =None, type: 'str' =None, updated_at: 'datetime' =None, user_id: 'str' =None):  # noqa: E501
+    def __init__(self, attached_at: 'datetime' =None, availability_zone: 'str' =None, detached_at: 'datetime' =None, encrypted: 'bool' =None, iops: 'str' =None, metadata: 'V1Metadata' =None, path: 'str' =None, provider: 'str' =None, provider_id: 'str' =None, region: 'str' =None, resource_id: 'str' =None, resource_type: 'str' =None, retention_period: 'str' =None, server_id: 'str' =None, size_gb: 'str' =None, state: 'V1VolumeState' =None, throughput: 'str' =None, type: 'str' =None, updated_at: 'datetime' =None, user_id: 'str' =None):  # noqa: E501
         """V1Volume - a model defined in Swagger"""  # noqa: E501
         self._attached_at = None
         self._availability_zone = None
-        self._cluster_id = None
-        self._created_at = None
         self._detached_at = None
         self._encrypted = None
-        self._id = None
         self._iops = None
+        self._metadata = None
         self._path = None
-        self._project_id = None
         self._provider = None
         self._provider_id = None
         self._region = None
         self._resource_id = None
         self._resource_type = None
+        self._retention_period = None
         self._server_id = None
         self._size_gb = None
+        self._state = None
         self._throughput = None
         self._type = None
         self._updated_at = None
@@ -116,22 +113,16 @@ class V1Volume(object):
             self.attached_at = attached_at
         if availability_zone is not None:
             self.availability_zone = availability_zone
-        if cluster_id is not None:
-            self.cluster_id = cluster_id
-        if created_at is not None:
-            self.created_at = created_at
         if detached_at is not None:
             self.detached_at = detached_at
         if encrypted is not None:
             self.encrypted = encrypted
-        if id is not None:
-            self.id = id
         if iops is not None:
             self.iops = iops
+        if metadata is not None:
+            self.metadata = metadata
         if path is not None:
             self.path = path
-        if project_id is not None:
-            self.project_id = project_id
         if provider is not None:
             self.provider = provider
         if provider_id is not None:
@@ -142,10 +133,14 @@ class V1Volume(object):
             self.resource_id = resource_id
         if resource_type is not None:
             self.resource_type = resource_type
+        if retention_period is not None:
+            self.retention_period = retention_period
         if server_id is not None:
             self.server_id = server_id
         if size_gb is not None:
             self.size_gb = size_gb
+        if state is not None:
+            self.state = state
         if throughput is not None:
             self.throughput = throughput
         if type is not None:
@@ -197,48 +192,6 @@ class V1Volume(object):
 
         self._availability_zone = availability_zone
 
-    @property
-    def cluster_id(self) -> 'str':
-        """Gets the cluster_id of this V1Volume.  # noqa: E501
-
-
-        :return: The cluster_id of this V1Volume.  # noqa: E501
-        :rtype: str
-        """
-        return self._cluster_id
-
-    @cluster_id.setter
-    def cluster_id(self, cluster_id: 'str'):
-        """Sets the cluster_id of this V1Volume.
-
-
-        :param cluster_id: The cluster_id of this V1Volume.  # noqa: E501
-        :type: str
-        """
-
-        self._cluster_id = cluster_id
-
-    @property
-    def created_at(self) -> 'datetime':
-        """Gets the created_at of this V1Volume.  # noqa: E501
-
-
-        :return: The created_at of this V1Volume.  # noqa: E501
-        :rtype: datetime
-        """
-        return self._created_at
-
-    @created_at.setter
-    def created_at(self, created_at: 'datetime'):
-        """Sets the created_at of this V1Volume.
-
-
-        :param created_at: The created_at of this V1Volume.  # noqa: E501
-        :type: datetime
-        """
-
-        self._created_at = created_at
-
     @property
     def detached_at(self) -> 'datetime':
         """Gets the detached_at of this V1Volume.  # noqa: E501
@@ -281,27 +234,6 @@ class V1Volume(object):
 
         self._encrypted = encrypted
 
-    @property
-    def id(self) -> 'str':
-        """Gets the id of this V1Volume.  # noqa: E501
-
-
-        :return: The id of this V1Volume.  # noqa: E501
-        :rtype: str
-        """
-        return self._id
-
-    @id.setter
-    def id(self, id: 'str'):
-        """Sets the id of this V1Volume.
-
-
-        :param id: The id of this V1Volume.  # noqa: E501
-        :type: str
-        """
-
-        self._id = id
-
     @property
     def iops(self) -> 'str':
         """Gets the iops of this V1Volume.  # noqa: E501
@@ -324,46 +256,46 @@ class V1Volume(object):
         self._iops = iops
 
     @property
-    def path(self) -> 'str':
-        """Gets the path of this V1Volume.  # noqa: E501
+    def metadata(self) -> 'V1Metadata':
+        """Gets the metadata of this V1Volume.  # noqa: E501
 
 
-        :return: The path of this V1Volume.  # noqa: E501
-        :rtype: str
+        :return: The metadata of this V1Volume.  # noqa: E501
+        :rtype: V1Metadata
         """
-        return self._path
+        return self._metadata
 
-    @path.setter
-    def path(self, path: 'str'):
-        """Sets the path of this V1Volume.
+    @metadata.setter
+    def metadata(self, metadata: 'V1Metadata'):
+        """Sets the metadata of this V1Volume.
 
 
-        :param path: The path of this V1Volume.  # noqa: E501
-        :type: str
+        :param metadata: The metadata of this V1Volume.  # noqa: E501
+        :type: V1Metadata
         """
 
-        self._path = path
+        self._metadata = metadata
 
     @property
-    def project_id(self) -> 'str':
-        """Gets the project_id of this V1Volume.  # noqa: E501
+    def path(self) -> 'str':
+        """Gets the path of this V1Volume.  # noqa: E501
 
 
-        :return: The project_id of this V1Volume.  # noqa: E501
+        :return: The path of this V1Volume.  # noqa: E501
         :rtype: str
         """
-        return self._project_id
+        return self._path
 
-    @project_id.setter
-    def project_id(self, project_id: 'str'):
-        """Sets the project_id of this V1Volume.
+    @path.setter
+    def path(self, path: 'str'):
+        """Sets the path of this V1Volume.
 
 
-        :param project_id: The project_id of this V1Volume.  # noqa: E501
+        :param path: The path of this V1Volume.  # noqa: E501
         :type: str
         """
 
-        self._project_id = project_id
+        self._path = path
 
     @property
     def provider(self) -> 'str':
@@ -470,6 +402,27 @@ class V1Volume(object):
 
         self._resource_type = resource_type
 
+    @property
+    def retention_period(self) -> 'str':
+        """Gets the retention_period of this V1Volume.  # noqa: E501
+
+
+        :return: The retention_period of this V1Volume.  # noqa: E501
+        :rtype: str
+        """
+        return self._retention_period
+
+    @retention_period.setter
+    def retention_period(self, retention_period: 'str'):
+        """Sets the retention_period of this V1Volume.
+
+
+        :param retention_period: The retention_period of this V1Volume.  # noqa: E501
+        :type: str
+        """
+
+        self._retention_period = retention_period
+
     @property
     def server_id(self) -> 'str':
         """Gets the server_id of this V1Volume.  # noqa: E501
@@ -512,6 +465,27 @@ class V1Volume(object):
 
         self._size_gb = size_gb
 
+    @property
+    def state(self) -> 'V1VolumeState':
+        """Gets the state of this V1Volume.  # noqa: E501
+
+
+        :return: The state of this V1Volume.  # noqa: E501
+        :rtype: V1VolumeState
+        """
+        return self._state
+
+    @state.setter
+    def state(self, state: 'V1VolumeState'):
+        """Sets the state of this V1Volume.
+
+
+        :param state: The state of this V1Volume.  # noqa: E501
+        :type: V1VolumeState
+        """
+
+        self._state = state
+
     @property
     def throughput(self) -> 'str':
         """Gets the throughput of this V1Volume.  # noqa: E501

lightning_sdk/lightning_cloud/openapi/models/v1_volume_state.py

@@ -0,0 +1,104 @@
+# coding: utf-8
+
+"""
+    external/v1/auth_service.proto
+
+    No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)  # noqa: E501
+
+    OpenAPI spec version: version not set
+    
+    Generated by: https://github.com/swagger-api/swagger-codegen.git
+
+    NOTE
+    ----
+    standard swagger-codegen-cli for this python client has been modified
+    by custom templates. The purpose of these templates is to include
+    typing information in the API and Model code. Please refer to the
+    main grid repository for more info
+"""
+
+import pprint
+import re  # noqa: F401
+
+from typing import TYPE_CHECKING
+
+import six
+
+if TYPE_CHECKING:
+    from datetime import datetime
+    from lightning_sdk.lightning_cloud.openapi.models import *
+
+class V1VolumeState(object):
+    """NOTE: This class is auto generated by the swagger code generator program.
+
+    Do not edit the class manually.
+    """
+
+    """
+    allowed enum values
+    """
+    UNSPECIFIED = "VOLUME_STATE_UNSPECIFIED"
+    ATTACHED = "VOLUME_STATE_ATTACHED"
+    DETACHED = "VOLUME_STATE_DETACHED"
+    """
+    Attributes:
+      swagger_types (dict): The key is attribute name
+                            and the value is attribute type.
+      attribute_map (dict): The key is attribute name
+                            and the value is json key in definition.
+    """
+    swagger_types = {
+    }
+
+    attribute_map = {
+    }
+
+    def __init__(self):  # noqa: E501
+        """V1VolumeState - a model defined in Swagger"""  # noqa: E501
+        self.discriminator = None
+
+    def to_dict(self) -> dict:
+        """Returns the model properties as a dict"""
+        result = {}
+
+        for attr, _ in six.iteritems(self.swagger_types):
+            value = getattr(self, attr)
+            if isinstance(value, list):
+                result[attr] = list(map(
+                    lambda x: x.to_dict() if hasattr(x, "to_dict") else x,
+                    value
+                ))
+            elif hasattr(value, "to_dict"):
+                result[attr] = value.to_dict()
+            elif isinstance(value, dict):
+                result[attr] = dict(map(
+                    lambda item: (item[0], item[1].to_dict())
+                    if hasattr(item[1], "to_dict") else item,
+                    value.items()
+                ))
+            else:
+                result[attr] = value
+        if issubclass(V1VolumeState, dict):
+            for key, value in self.items():
+                result[key] = value
+
+        return result
+
+    def to_str(self) -> str:
+        """Returns the string representation of the model"""
+        return pprint.pformat(self.to_dict())
+
+    def __repr__(self) -> str:
+        """For `print` and `pprint`"""
+        return self.to_str()
+
+    def __eq__(self, other: 'V1VolumeState') -> bool:
+        """Returns true if both objects are equal"""
+        if not isinstance(other, V1VolumeState):
+            return False
+
+        return self.__dict__ == other.__dict__
+
+    def __ne__(self, other: 'V1VolumeState') -> bool:
+        """Returns true if both objects are not equal"""
+        return not self == other

lightning_sdk/pipeline/__init__.py

@@ -1,5 +1,14 @@
+from lightning_sdk import Studio
 from lightning_sdk.pipeline.pipeline import Pipeline
 from lightning_sdk.pipeline.schedule import Schedule
-from lightning_sdk.pipeline.types import MMT, Deployment, Job
+from lightning_sdk.pipeline.steps import DeploymentReleaseStep, DeploymentStep, JobStep, MMTStep
 
-__all__ = ["Pipeline", "Job", "MMT", "Deployment", "Schedule"]
+__all__ = [
+    "Pipeline",
+    "JobStep",
+    "MMTStep",
+    "DeploymentStep",
+    "Schedule",
+    "Studio",
+    "DeploymentReleaseStep",
+]

lightning_sdk/pipeline/pipeline.py

@@ -1,3 +1,4 @@
+import os
 from typing import TYPE_CHECKING, List, Optional, Union
 
 from lightning_sdk.api import UserApi
@@ -5,9 +6,10 @@ from lightning_sdk.api.pipeline_api import PipelineApi
 from lightning_sdk.lightning_cloud.login import Auth
 from lightning_sdk.organization import Organization
 from lightning_sdk.pipeline.printer import PipelinePrinter
-from lightning_sdk.pipeline.types import MMT, Deployment, Job
+from lightning_sdk.pipeline.steps import DeploymentStep, JobStep, MMTStep, _get_studio
 from lightning_sdk.pipeline.utils import prepare_steps
 from lightning_sdk.services.utilities import _get_cluster
+from lightning_sdk.studio import Studio
 from lightning_sdk.teamspace import Teamspace
 from lightning_sdk.user import User
 from lightning_sdk.utils.resolve import _resolve_org, _resolve_teamspace, _resolve_user
@@ -25,6 +27,7 @@ class Pipeline:
         user: Union[str, "User", None] = None,
         cloud_account: Optional[str] = None,
         shared_filesystem: Optional[bool] = None,
+        studio: Optional[Union[Studio, str]] = None,
     ) -> None:
         """The Lightning Pipeline can be used to create complex DAG.
 
@@ -58,10 +61,12 @@ class Pipeline:
         )
 
         self._pipeline_api = PipelineApi()
-        self._cloud_account = _get_cluster(
+        self._cloud_account = cloud_account
+        self._default_cluster = _get_cluster(
             client=self._pipeline_api._client, project_id=self._teamspace.id, cluster_id=cloud_account
         )
-        self._shared_filesystem = shared_filesystem
+        self._shared_filesystem = shared_filesystem if shared_filesystem is not None else True
+        self._studio = _get_studio(studio)
         self._is_created = False
 
         pipeline = None
@@ -75,18 +80,29 @@ class Pipeline:
         else:
             self._pipeline = None
 
-    def run(self, steps: List[Union[Job, Deployment, MMT]], schedules: Optional[List["Schedule"]] = None) -> None:
+    def run(
+        self, steps: List[Union[JobStep, DeploymentStep, MMTStep]], schedules: Optional[List["Schedule"]] = None
+    ) -> None:
         if len(steps) == 0:
             raise ValueError("The provided steps is empty")
 
-        for step_idx, step in enumerate(steps):
-            if step.name in [None, ""]:
-                raise ValueError(f"The step {step_idx} requires a name")
+        for step_idx, pipeline_step in enumerate(steps):
+            if pipeline_step.name in [None, ""]:
+                pipeline_step.name = f"step-{step_idx}"
+
+            if (
+                self._studio is not None
+                and (pipeline_step.image == "" or pipeline_step.image is None)
+                and pipeline_step.studio is None
+            ):
+                pipeline_step.cloud_account = self._studio.cloud_account
+                pipeline_step.studio = self._studio
+
+        cluster_ids = set(step.cloud_account for step in steps if step.cloud_account not in ["", None])  # noqa: C401
+
+        cloud_account = list(cluster_ids)[0] if len(cluster_ids) == 1 and self._cloud_account is None else ""  # noqa: RUF015
 
-        steps = [
-            step.to_proto(self._teamspace, self._cloud_account.cluster_id or "", self._shared_filesystem)
-            for step in steps
-        ]
+        steps = [step.to_proto(self._teamspace, cloud_account, self._shared_filesystem) for step in steps]
 
         proto_steps = prepare_steps(steps)
         schedules = schedules or []
@@ -97,13 +113,18 @@ class Pipeline:
             self._name,
             self._teamspace.id,
             proto_steps,
-            self._shared_filesystem or False,
+            self._shared_filesystem,
             schedules,
             parent_pipeline_id,
         )
 
         printer = PipelinePrinter(
-            self._name, parent_pipeline_id is None, self._pipeline, self._teamspace, proto_steps, schedules
+            self._name,
+            parent_pipeline_id is None,
+            self._pipeline,
+            self._teamspace,
+            proto_steps,
+            schedules,
         )
         printer.print_summary()
 
@@ -124,3 +145,7 @@ class Pipeline:
         if self._pipeline:
             return self._pipeline.name
         return None
+
+    @classmethod
+    def from_env(cls) -> "Pipeline":
+        return Pipeline(name=os.getenv("LIGHTNING_PIPELINE_ID", ""))

lightning_sdk/pipeline/printer.py

@@ -1,7 +1,7 @@
 import os
 from typing import Any, ClassVar, Dict, List
 
-from lightning_sdk.lightning_cloud.openapi.models import V1JobSpec, V1PipelineStepType
+from lightning_sdk.lightning_cloud.openapi.models import V1JobSpec, V1Pipeline, V1PipelineStepType
 
 
 class PipelinePrinter:
@@ -14,14 +14,26 @@ class PipelinePrinter:
     }
 
     def __init__(
-        self, name: str, initial: bool, pipeline: Any, teamspace: Any, proto_steps: List[Any], schedules: List[Any]
+        self,
+        name: str,
+        initial: bool,
+        pipeline: V1Pipeline,
+        teamspace: Any,
+        proto_steps: List[Any],
+        schedules: List[Any],
     ) -> None:
         self._name = name
         self._initial = initial
         self._pipeline = pipeline
         self._teamspace = teamspace
         self._proto_steps = proto_steps
+        self._shared_filesystem = pipeline.shared_filesystem
         self._schedules = schedules
+        cluster_ids: set[str] = set()
+        for step in self._proto_steps:
+            job_spec = self._get_spec(step)
+            cluster_ids.add(job_spec.cluster_id)
+        self._cluster_ids = cluster_ids
 
     def print_summary(self) -> None:
         """Prints the full, formatted summary of the created pipeline."""
@@ -32,6 +44,7 @@ class PipelinePrinter:
         self._print_steps()
         self._print_schedules()
         self._print_cloud_account()
+        self._print_shared_filesystem()
         self._print_footer()
 
     def _print(self, value: str) -> None:
@@ -75,7 +88,7 @@ class PipelinePrinter:
         team: str = self._teamspace.name
         pipeline_name: str = self._name
 
-        pipeline_url = f"{cloud_url}/{owner}/{team}/pipelines/{pipeline_name}?app_id=pipeline&section=Graph"
+        pipeline_url = f"{cloud_url}/{owner}/{team}/pipelines/{pipeline_name}?app_id=pipeline"
 
         self._print("\n" + "─" * 60)
         self._print(f"🔗 View your pipeline in the browser:\n   {pipeline_url}")
@@ -85,15 +98,21 @@ class PipelinePrinter:
         if not self._proto_steps:
             return
 
-        cluster_ids: set[str] = set()
-        for step in self._proto_steps:
-            job_spec = self._get_spec(step)
-            cluster_ids.add(job_spec.cluster_id)
-
-        self._print(f"\nCloud account{'s' if len(cluster_ids) > 1 else ''}:")
-        for cluster_id in sorted(cluster_ids):
+        self._print(f"\nCloud account{'s' if len(self._cluster_ids) > 1 else ''}:")
+        for cluster_id in sorted(self._cluster_ids):
             self._print(f"  - {cluster_id}")
 
+    def _print_shared_filesystem(self) -> None:
+        self._print(f"\nShared filesystem: {self._shared_filesystem.enabled}")
+
+        if self._shared_filesystem.enabled and len(self._cluster_ids) == 1:
+            shared_path = ""
+            if self._pipeline.shared_filesystem.s3_folder:
+                cluster_id = list(self._cluster_ids)[0]  # noqa: RUF015
+                shared_path = f"/teamspace/s3_folders/pipelines-{cluster_id}"
+            if shared_path:
+                self._print(f"  - {shared_path}")
+
     def _get_spec(self, step: Any) -> V1JobSpec:
         if step.type == V1PipelineStepType.DEPLOYMENT:
             return step.deployment.spec

lightning_sdk/pipeline/schedule.py

@@ -1,7 +1,8 @@
 from dataclasses import dataclass
+from typing import Optional
 
 
 @dataclass
 class Schedule:
-    name: str
     cron_expression: str
+    name: Optional[str] = None