luminarycloud 0.22.2__py3-none-any.whl → 0.23.0__py3-none-any.whl

luminarycloud/pipelines/api.py

@@ -1,5 +1,5 @@
-# Copyright 2023-2024 Luminary Cloud, Inc. All Rights Reserved.
-from typing import Any, Literal
+# Copyright 2023-2025 Luminary Cloud, Inc. All Rights Reserved.
+from typing import Generic, Iterator, Literal, TypeVar
 from dataclasses import dataclass
 
 from datetime import datetime
@@ -7,13 +7,78 @@ from time import time, sleep
 import logging
 
 from .arguments import PipelineArgValueType
-from .core import Stage
 from ..pipelines import Pipeline, PipelineArgs
 from .._client import get_default_client
 from .._helpers import parse_iso_datetime
 
 logger = logging.getLogger(__name__)
 
+T = TypeVar("T")
+
+
+class PipelinesPaginationIterator(Generic[T]):
+    """
+    Iterator for pipelines API endpoints that use offset-based pagination (skip/limit).
+
+    This is different from the token-based PaginationIterator in _helpers.pagination.
+    Lazily fetches pages as needed and yields individual items.
+    """
+
+    def __init__(self, http_client, endpoint: str, model_class, page_size: int = 100):
+        """
+        Parameters
+        ----------
+        http_client
+            HTTP client to use for requests
+        endpoint : str
+            API endpoint to fetch from
+        model_class
+            Class to use for deserializing items (must have from_json classmethod)
+        page_size : int
+            Number of items to fetch per page (default: 100)
+        """
+        self.http_client = http_client
+        self.endpoint = endpoint
+        self.model_class = model_class
+        self.page_size = page_size
+        self._skip = 0
+        self._current_page: list[T] | None = None
+        self._exhausted = False
+
+    def __iter__(self) -> "PipelinesPaginationIterator[T]":
+        return self
+
+    def __next__(self) -> T:
+        if self._current_page is None:
+            self._fetch_next_page()
+
+        # mypy needs this assertion even though _current_page can't be None here
+        assert self._current_page is not None
+
+        if len(self._current_page) == 0:
+            if self._exhausted:
+                raise StopIteration
+            self._fetch_next_page()
+            # After fetching, check if we're still empty (means we hit the end)
+            if len(self._current_page) == 0:
+                raise StopIteration
+
+        return self._current_page.pop(0)
+
+    def _fetch_next_page(self) -> None:
+        """Fetch the next page of results from the API."""
+        response = self.http_client.get(
+            self.endpoint, params={"skip": self._skip, "limit": self.page_size}
+        )
+        items = response["data"]
+
+        self._current_page = [self.model_class.from_json(item) for item in items]
+
+        self._skip += len(items)
+
+        if len(items) < self.page_size:
+            self._exhausted = True
+
 
 @dataclass
 class LogLine:
@@ -30,6 +95,50 @@ class LogLine:
         )
 
 
+@dataclass
+class StageDefinition:
+    """
+    Represents a stage definition from a pipeline.
+    """
+
+    id: str
+    name: str
+    stage_type: str
+
+    @classmethod
+    def from_json(cls, json: dict) -> "StageDefinition":
+        return cls(
+            id=json["id"],
+            name=json["name"],
+            stage_type=json["stage_type"],
+        )
+
+
+@dataclass
+class PipelineTaskRecord:
+    """
+    A PipelineTaskRecord represents a task within a pipeline job run.
+    """
+
+    status: Literal["pending", "running", "completed", "failed", "upstream_failed", "cancelled"]
+    artifacts: dict[str, dict]
+    stage: StageDefinition | None
+    created_at: datetime
+    updated_at: datetime
+    error_messages: list[str] | None
+
+    @classmethod
+    def from_json(cls, json: dict) -> "PipelineTaskRecord":
+        return cls(
+            status=json["status"],
+            artifacts=json["artifacts"],
+            stage=StageDefinition.from_json(json["stage"]) if json.get("stage") else None,
+            created_at=parse_iso_datetime(json["created_at"]),
+            updated_at=parse_iso_datetime(json["updated_at"]),
+            error_messages=json.get("error_messages"),
+        )
+
+
 @dataclass
 class PipelineRecord:
     """
@@ -59,17 +168,36 @@ class PipelineRecord:
             updated_at=parse_iso_datetime(json["updated_at"]),
         )
 
-    def pipeline_jobs(self) -> "list[PipelineJobRecord]":
+    def pipeline_jobs(self, page_size: int = 100) -> Iterator["PipelineJobRecord"]:
         """
-        Returns a list of pipeline jobs that were created from this pipeline.
+        Returns an iterator of pipeline jobs that were created from this pipeline.
+
+        The iterator will automatically fetch additional pages as needed.
+
+        Parameters
+        ----------
+        page_size : int, optional
+            Number of pipeline jobs to fetch per page. Defaults to 100, max is 1000.
 
         Returns
         -------
-        list[PipelineJobRecord]
-            A list of PipelineJobRecord objects.
+        Iterator[PipelineJobRecord]
+            An iterator of PipelineJobRecord objects.
+
+        Examples
+        --------
+        >>> pipeline = pipelines.get_pipeline("pipeline-123")
+        >>> for job in pipeline.pipeline_jobs():
+        ...     print(job.name)
+        >>> # Or convert to a list if needed:
+        >>> all_jobs = list(pipeline.pipeline_jobs())
         """
-        res = get_default_client().http.get(f"/rest/v0/pipelines/{self.id}/pipeline_jobs")
-        return [PipelineJobRecord.from_json(p) for p in res["data"]]
+        return PipelinesPaginationIterator(
+            get_default_client().http,
+            f"/rest/v0/pipelines/{self.id}/pipeline_jobs",
+            PipelineJobRecord,
+            page_size,
+        )
 
     def delete(self) -> None:
         """
@@ -358,14 +486,17 @@ class PipelineJobRunRecord:
     idx: int
     arguments: list[PipelineArgValueType]
     status: Literal["pending", "running", "completed", "failed", "cancelled"]
+    tasks: list[PipelineTaskRecord]
 
     @classmethod
     def from_json(cls, json: dict) -> "PipelineJobRunRecord":
+        tasks_json = json.get("tasks", [])
         return cls(
             pipeline_job_id=json["pipeline_job_id"],
             idx=json["idx"],
             arguments=json["arguments"],
             status=json["status"],
+            tasks=[PipelineTaskRecord.from_json(t) for t in tasks_json],
         )
 
     def pipeline_job(self) -> PipelineJobRecord:
@@ -443,12 +574,34 @@ def create_pipeline(
     return PipelineRecord.from_json(res["data"])
 
 
-def list_pipelines() -> list[PipelineRecord]:
+def iterate_pipelines(page_size: int = 100) -> Iterator[PipelineRecord]:
     """
-    List all pipelines.
+    Iterate over all pipelines with pagination.
+
+    Parameters
+    ----------
+    page_size : int, optional
+        Number of pipelines to fetch per page. Defaults to 100, max is 1000.
+
+    Returns
+    -------
+    Iterator[PipelineRecord]
+        An iterator that yields PipelineRecord objects one at a time.
+
+    Examples
+    --------
+    Iterate over all pipelines:
+
+    >>> for pipeline in pipelines.iterate_pipelines():
+    ...     print(pipeline.name)
+
+    Convert to a list if needed:
+
+    >>> all_pipelines = list(pipelines.iterate_pipelines())
     """
-    res = get_default_client().http.get("/rest/v0/pipelines")
-    return [PipelineRecord.from_json(p) for p in res["data"]]
+    return PipelinesPaginationIterator(
+        get_default_client().http, "/rest/v0/pipelines", PipelineRecord, page_size
+    )
 
 
 def get_pipeline(id: str) -> PipelineRecord:
@@ -511,9 +664,31 @@ def get_pipeline_job(id: str) -> PipelineJobRecord:
     return PipelineJobRecord.from_json(res["data"])
 
 
-def list_pipeline_jobs() -> list[PipelineJobRecord]:
+def iterate_pipeline_jobs(page_size: int = 100) -> Iterator[PipelineJobRecord]:
     """
-    List all pipeline jobs.
+    Iterate over all pipeline jobs with pagination.
+
+    Parameters
+    ----------
+    page_size : int, optional
+        Number of pipeline jobs to fetch per page. Defaults to 100, max is 1000.
+
+    Returns
+    -------
+    Iterator[PipelineJobRecord]
+        An iterator that yields PipelineJobRecord objects one at a time.
+
+    Examples
+    --------
+    Iterate over all pipeline jobs:
+
+    >>> for job in pipelines.iterate_pipeline_jobs():
+    ...     print(job.name)
+
+    Convert to a list if needed:
+
+    >>> all_jobs = list(pipelines.iterate_pipeline_jobs())
     """
-    res = get_default_client().http.get("/rest/v0/pipeline_jobs")
-    return [PipelineJobRecord.from_json(p) for p in res["data"]]
+    return PipelinesPaginationIterator(
+        get_default_client().http, "/rest/v0/pipeline_jobs", PipelineJobRecord, page_size
+    )

luminarycloud/pipelines/arguments.py

@@ -28,10 +28,15 @@ class _NVS(PipelineParameter):
         return isinstance(value, str) and value.startswith("namedvarset-")
 
 
-ArgNamedVariableSet = _NVS("$named-variable-set")
+PP_NAMED_VARIABLE_SET_ID = _NVS("$named-variable-set")
 """
-This can be used in a PipelineArgs params list to add a Named Variable Set column to the args table.
-There must be zero or one of these in a PipelineArgs params list.
+A constant PipelineParameter that can be used in a PipelineArgs params list to add a Named Variable
+Set ID column to the args table.  There must be zero or one of these in a PipelineArgs params list.
+"""
+
+ArgNamedVariableSet = PP_NAMED_VARIABLE_SET_ID
+"""
+Deprecated alias for PP_NAMED_VARIABLE_SET_ID.
 """
 
 # The types that are allowed as PipelineArgs values. This is a union of all concrete