luminarycloud 0.22.3__py3-none-any.whl → 0.23.1__py3-none-any.whl

luminarycloud/physics_ai/training_jobs.py

@@ -1,20 +1,33 @@
 # Copyright 2025 Luminary Cloud, Inc. All Rights Reserved.
-from typing import List, Optional
+from typing import List
 from datetime import datetime
 
 from .._client import get_default_client
 from .._proto.api.v0.luminarycloud.physics_ai import physics_ai_pb2 as physaipb
 from .._proto.base import base_pb2 as basepb
 from .._wrapper import ProtoWrapper, ProtoWrapperBase
-from ..types.ids import PhysicsAiArchitectureVersionID
+from ..types.ids import PhysicsAiArchitectureVersionID, PhysicsAiTrainingJobID
+
+
+def get_training_job(training_job_id: PhysicsAiTrainingJobID) -> "PhysicsAiTrainingJob":
+    """
+    Get a Physics AI training job by ID.
+
+    Args:
+        training_job_id: The ID of the training job to retrieve.
+
+    Returns:
+        The training job.
+    """
+    req = physaipb.GetTrainingJobRequest(training_job_id=training_job_id)
+    resp = get_default_client().GetTrainingJob(req)
+    return PhysicsAiTrainingJob(resp.training_job)
 
 
 @ProtoWrapper(physaipb.PhysicsAiTrainingJob)
 class PhysicsAiTrainingJob(ProtoWrapperBase):
     """
     Represents a Physics AI training job.
-
-    .. warning:: This feature is experimental and may change or be removed without notice.
     """
 
     id: str
@@ -24,6 +37,7 @@ class PhysicsAiTrainingJob(ProtoWrapperBase):
     training_data_source_type: physaipb.TrainingDataSourceType
     training_description: str
     external_dataset_uri: str
+    dataset_id: str
     initialization_type: physaipb.ModelInitializationType
     base_model_version_id: str
     status: basepb.JobStatus
@@ -35,3 +49,35 @@ class PhysicsAiTrainingJob(ProtoWrapperBase):
 
     def get_status(self) -> str:
         return basepb.JobStatusType.Name(self.status.typ)
+
+    def cancel(self) -> None:
+        """
+        Cancel this training job.
+
+        This method requests cancellation of the training job. If the job is already
+        suspended or suspending, this is a no-op.
+        """
+        req = physaipb.CancelTrainingJobRequest(training_job_id=self.id)
+        get_default_client().CancelTrainingJob(req)
+
+
+def list_training_jobs() -> List[PhysicsAiTrainingJob]:
+    """
+    List Physics AI training jobs for the current user.
+
+    Returns
+    -------
+    List[PhysicsAiTrainingJob]
+        A list of training jobs, ordered by creation time (newest first).
+
+    Examples
+    --------
+    List all training jobs:
+
+    >>> jobs = list_training_jobs()
+    >>> for job in jobs:
+    ...     print(f"{job.id}: {job.get_status()}")
+    """
+    req = physaipb.ListTrainingJobsRequest()
+    response = get_default_client().ListTrainingJobs(req)
+    return [PhysicsAiTrainingJob(job) for job in response.training_jobs]

luminarycloud/pipelines/__init__.py

@@ -27,16 +27,17 @@ from .stages import (
     ReadGeometryOutputs as ReadGeometryOutputs,
     ReadMesh as ReadMesh,
     ReadMeshOutputs as ReadMeshOutputs,
-    ModifyGeometry as ModifyGeometry,
-    ModifyGeometryOutputs as ModifyGeometryOutputs,
     Mesh as Mesh,
     MeshOutputs as MeshOutputs,
+    WaitForMesh as WaitForMesh,
+    WaitForMeshOutputs as WaitForMeshOutputs,
     Simulate as Simulate,
     SimulateOutputs as SimulateOutputs,
 )
 
 from .arguments import (
     PipelineArgs as PipelineArgs,
+    PP_NAMED_VARIABLE_SET_ID as PP_NAMED_VARIABLE_SET_ID,
     ArgNamedVariableSet as ArgNamedVariableSet,
 )
 
@@ -47,11 +48,11 @@ from .flowables import (
 
 from .api import (
     create_pipeline as create_pipeline,
-    list_pipelines as list_pipelines,
+    iterate_pipelines as iterate_pipelines,
     get_pipeline as get_pipeline,
     create_pipeline_job as create_pipeline_job,
     get_pipeline_job as get_pipeline_job,
-    list_pipeline_jobs as list_pipeline_jobs,
+    iterate_pipeline_jobs as iterate_pipeline_jobs,
     PipelineJobRecord as PipelineJobRecord,
     PipelineRecord as PipelineRecord,
     PipelineJobRunRecord as PipelineJobRunRecord,

luminarycloud/pipelines/api.py

@@ -1,5 +1,5 @@
-# Copyright 2023-2024 Luminary Cloud, Inc. All Rights Reserved.
-from typing import Any, Literal
+# Copyright 2023-2025 Luminary Cloud, Inc. All Rights Reserved.
+from typing import Generic, Iterator, Literal, TypeVar
 from dataclasses import dataclass
 
 from datetime import datetime
@@ -7,13 +7,78 @@ from time import time, sleep
 import logging
 
 from .arguments import PipelineArgValueType
-from .core import Stage
 from ..pipelines import Pipeline, PipelineArgs
 from .._client import get_default_client
 from .._helpers import parse_iso_datetime
 
 logger = logging.getLogger(__name__)
 
+T = TypeVar("T")
+
+
+class PipelinesPaginationIterator(Generic[T]):
+    """
+    Iterator for pipelines API endpoints that use offset-based pagination (skip/limit).
+
+    This is different from the token-based PaginationIterator in _helpers.pagination.
+    Lazily fetches pages as needed and yields individual items.
+    """
+
+    def __init__(self, http_client, endpoint: str, model_class, page_size: int = 100):
+        """
+        Parameters
+        ----------
+        http_client
+            HTTP client to use for requests
+        endpoint : str
+            API endpoint to fetch from
+        model_class
+            Class to use for deserializing items (must have from_json classmethod)
+        page_size : int
+            Number of items to fetch per page (default: 100)
+        """
+        self.http_client = http_client
+        self.endpoint = endpoint
+        self.model_class = model_class
+        self.page_size = page_size
+        self._skip = 0
+        self._current_page: list[T] | None = None
+        self._exhausted = False
+
+    def __iter__(self) -> "PipelinesPaginationIterator[T]":
+        return self
+
+    def __next__(self) -> T:
+        if self._current_page is None:
+            self._fetch_next_page()
+
+        # mypy needs this assertion even though _current_page can't be None here
+        assert self._current_page is not None
+
+        if len(self._current_page) == 0:
+            if self._exhausted:
+                raise StopIteration
+            self._fetch_next_page()
+            # After fetching, check if we're still empty (means we hit the end)
+            if len(self._current_page) == 0:
+                raise StopIteration
+
+        return self._current_page.pop(0)
+
+    def _fetch_next_page(self) -> None:
+        """Fetch the next page of results from the API."""
+        response = self.http_client.get(
+            self.endpoint, params={"skip": self._skip, "limit": self.page_size}
+        )
+        items = response["data"]
+
+        self._current_page = [self.model_class.from_json(item) for item in items]
+
+        self._skip += len(items)
+
+        if len(items) < self.page_size:
+            self._exhausted = True
+
 
 @dataclass
 class LogLine:
@@ -103,17 +168,36 @@ class PipelineRecord:
             updated_at=parse_iso_datetime(json["updated_at"]),
         )
 
-    def pipeline_jobs(self) -> "list[PipelineJobRecord]":
+    def pipeline_jobs(self, page_size: int = 100) -> Iterator["PipelineJobRecord"]:
         """
-        Returns a list of pipeline jobs that were created from this pipeline.
+        Returns an iterator of pipeline jobs that were created from this pipeline.
+
+        The iterator will automatically fetch additional pages as needed.
+
+        Parameters
+        ----------
+        page_size : int, optional
+            Number of pipeline jobs to fetch per page. Defaults to 100, max is 1000.
 
         Returns
         -------
-        list[PipelineJobRecord]
-            A list of PipelineJobRecord objects.
+        Iterator[PipelineJobRecord]
+            An iterator of PipelineJobRecord objects.
+
+        Examples
+        --------
+        >>> pipeline = pipelines.get_pipeline("pipeline-123")
+        >>> for job in pipeline.pipeline_jobs():
+        ...     print(job.name)
+        >>> # Or convert to a list if needed:
+        >>> all_jobs = list(pipeline.pipeline_jobs())
         """
-        res = get_default_client().http.get(f"/rest/v0/pipelines/{self.id}/pipeline_jobs")
-        return [PipelineJobRecord.from_json(p) for p in res["data"]]
+        return PipelinesPaginationIterator(
+            get_default_client().http,
+            f"/rest/v0/pipelines/{self.id}/pipeline_jobs",
+            PipelineJobRecord,
+            page_size,
+        )
 
     def delete(self) -> None:
         """
@@ -402,14 +486,17 @@ class PipelineJobRunRecord:
     idx: int
     arguments: list[PipelineArgValueType]
     status: Literal["pending", "running", "completed", "failed", "cancelled"]
+    tasks: list[PipelineTaskRecord]
 
     @classmethod
     def from_json(cls, json: dict) -> "PipelineJobRunRecord":
+        tasks_json = json.get("tasks", [])
         return cls(
             pipeline_job_id=json["pipeline_job_id"],
             idx=json["idx"],
             arguments=json["arguments"],
             status=json["status"],
+            tasks=[PipelineTaskRecord.from_json(t) for t in tasks_json],
         )
 
     def pipeline_job(self) -> PipelineJobRecord:
@@ -458,23 +545,6 @@ class PipelineJobRunRecord:
         )
         return res["data"]
 
-    def tasks(self) -> list[PipelineTaskRecord]:
-        """
-        Returns a list of tasks for this pipeline job run.
-
-        Each task represents an execution of a stage of the pipeline, with its own
-        status, artifacts, and stage information.
-
-        Returns
-        -------
-        list[PipelineTaskRecord]
-            The tasks associated with this pipeline job run.
-        """
-        res = get_default_client().http.get(
-            f"/rest/v0/pipeline_jobs/{self.pipeline_job_id}/runs/{self.idx}/tasks"
-        )
-        return [PipelineTaskRecord.from_json(t) for t in res["data"]]
-
 
 def create_pipeline(
     name: str, pipeline: Pipeline | str, description: str | None = None
@@ -504,12 +574,34 @@ def create_pipeline(
     return PipelineRecord.from_json(res["data"])
 
 
-def list_pipelines() -> list[PipelineRecord]:
+def iterate_pipelines(page_size: int = 100) -> Iterator[PipelineRecord]:
     """
-    List all pipelines.
+    Iterate over all pipelines with pagination.
+
+    Parameters
+    ----------
+    page_size : int, optional
+        Number of pipelines to fetch per page. Defaults to 100, max is 1000.
+
+    Returns
+    -------
+    Iterator[PipelineRecord]
+        An iterator that yields PipelineRecord objects one at a time.
+
+    Examples
+    --------
+    Iterate over all pipelines:
+
+    >>> for pipeline in pipelines.iterate_pipelines():
+    ...     print(pipeline.name)
+
+    Convert to a list if needed:
+
+    >>> all_pipelines = list(pipelines.iterate_pipelines())
     """
-    res = get_default_client().http.get("/rest/v0/pipelines")
-    return [PipelineRecord.from_json(p) for p in res["data"]]
+    return PipelinesPaginationIterator(
+        get_default_client().http, "/rest/v0/pipelines", PipelineRecord, page_size
+    )
 
 
 def get_pipeline(id: str) -> PipelineRecord:
@@ -572,9 +664,31 @@ def get_pipeline_job(id: str) -> PipelineJobRecord:
     return PipelineJobRecord.from_json(res["data"])
 
 
-def list_pipeline_jobs() -> list[PipelineJobRecord]:
+def iterate_pipeline_jobs(page_size: int = 100) -> Iterator[PipelineJobRecord]:
     """
-    List all pipeline jobs.
+    Iterate over all pipeline jobs with pagination.
+
+    Parameters
+    ----------
+    page_size : int, optional
+        Number of pipeline jobs to fetch per page. Defaults to 100, max is 1000.
+
+    Returns
+    -------
+    Iterator[PipelineJobRecord]
+        An iterator that yields PipelineJobRecord objects one at a time.
+
+    Examples
+    --------
+    Iterate over all pipeline jobs:
+
+    >>> for job in pipelines.iterate_pipeline_jobs():
+    ...     print(job.name)
+
+    Convert to a list if needed:
+
+    >>> all_jobs = list(pipelines.iterate_pipeline_jobs())
     """
-    res = get_default_client().http.get("/rest/v0/pipeline_jobs")
-    return [PipelineJobRecord.from_json(p) for p in res["data"]]
+    return PipelinesPaginationIterator(
+        get_default_client().http, "/rest/v0/pipeline_jobs", PipelineJobRecord, page_size
+    )

luminarycloud/pipelines/arguments.py

@@ -28,10 +28,15 @@ class _NVS(PipelineParameter):
         return isinstance(value, str) and value.startswith("namedvarset-")
 
 
-ArgNamedVariableSet = _NVS("$named-variable-set")
+PP_NAMED_VARIABLE_SET_ID = _NVS("$named-variable-set")
 """
-This can be used in a PipelineArgs params list to add a Named Variable Set column to the args table.
-There must be zero or one of these in a PipelineArgs params list.
+A constant PipelineParameter that can be used in a PipelineArgs params list to add a Named Variable
+Set ID column to the args table.  There must be zero or one of these in a PipelineArgs params list.
+"""
+
+ArgNamedVariableSet = PP_NAMED_VARIABLE_SET_ID
+"""
+Deprecated alias for PP_NAMED_VARIABLE_SET_ID.
 """
 
 # The types that are allowed as PipelineArgs values. This is a union of all concrete

luminarycloud/pipelines/core.py

@@ -3,7 +3,9 @@ from __future__ import annotations
 from abc import ABC, abstractmethod
 from dataclasses import is_dataclass, fields
 from typing import Any, Callable, Mapping, Type, TypeVar, Generic, TYPE_CHECKING
+from .user_code_validation import validate_portable_top_level_function
 from typing_extensions import Self
+import builtins
 import inspect
 import re
 import textwrap
@@ -330,7 +332,7 @@ class RunScript:
 
     def __init__(
         self,
-        script: Callable[..., dict[str, Any]] | str,
+        script: Callable[..., dict[str, Any] | None] | str,
         *,
         stage_name: str | None = None,
         inputs: dict[str, PipelineOutput] | None = None,
@@ -347,11 +349,11 @@ class RunScript:
             raise ValueError(f"RunScript params and inputs cannot share names: {overlap}")
 
         inputs_and_params = set(inputs.keys()).union(params.keys())
-        script_source, callable_entrypoint = self._get_script_source(script, inputs_and_params)
-        self._stage_type_name = "RunScript"
-        self._entrypoint = (
-            entrypoint or callable_entrypoint or self._infer_entrypoint(script_source)
+        validated_source = validate_portable_top_level_function(
+            script, must_have_params=inputs_and_params, can_have_params={"context"}
         )
+        self._stage_type_name = "RunScript"
+        self._entrypoint = entrypoint or validated_source.entrypoint
         self._name = (
             stage_name if stage_name is not None else self._default_stage_name(self._entrypoint)
         )
@@ -381,7 +383,7 @@ class RunScript:
         self.outputs = DynamicStageOutputs(self, output_flowable_types)
 
         reserved_params = {
-            "$script": script_source,
+            "$script": validated_source.source,
             "$output_types": {name: ft.value for name, ft in output_flowable_types.items()},
             "$entrypoint": self._entrypoint,
         }
@@ -414,8 +416,6 @@ class RunScript:
         output_types: Mapping[str, type[PipelineOutput] | str | FlowableType],
     ) -> dict[str, FlowableType]:
         normalized: dict[str, FlowableType] = {}
-        if not output_types:
-            raise ValueError("RunScript stages must declare at least one output")
         for name, value in output_types.items():
             if isinstance(value, FlowableType):
                 normalized[name] = value
@@ -429,68 +429,6 @@ class RunScript:
                 )
         return normalized
 
-    @staticmethod
-    def _validate_script(
-        script: Callable[..., dict[str, Any]], inputs_and_params: set[str]
-    ) -> None:
-        closurevars = inspect.getclosurevars(script)
-        if closurevars.nonlocals:
-            raise ValueError(
-                f"RunScript functions must not close over non-local variables. Found these non-local variables: {', '.join(closurevars.nonlocals.keys())}"
-            )
-        globals_except_lc = {
-            k for k in closurevars.globals.keys() if k != "lc" and k != "luminarycloud"
-        }
-        if globals_except_lc:
-            raise ValueError(
-                f"RunScript functions must not rely on global variables, including imports. All modules your script needs (except `luminarycloud` or `lc`) must be imported in the function body. Found globals: {', '.join(globals_except_lc)}"
-            )
-        script_params = set(inspect.signature(script).parameters.keys())
-        if script_params != inputs_and_params and script_params != inputs_and_params | {"context"}:
-            raise ValueError(
-                f"RunScript function must take exactly the same parameters as the inputs and params (and optionally `context`): {script_params} != {inputs_and_params}"
-            )
-
-    @staticmethod
-    def _get_script_source(
-        script: Callable[..., dict[str, Any]] | str,
-        inputs_and_params: set[str],
-    ) -> tuple[str, str | None]:
-        if callable(script):
-            RunScript._validate_script(script, inputs_and_params)
-            try:
-                source_lines, _ = inspect.getsourcelines(script)  # type: ignore[arg-type]
-            except (OSError, IOError, TypeError) as exc:
-                raise ValueError(f"Unable to retrieve source for {script.__name__}: {exc}") from exc
-            # Drop decorator lines (everything before the `def`)
-            for i, line in enumerate(source_lines):
-                if line.lstrip().startswith("def "):
-                    source_lines = source_lines[i:]
-                    break
-            source = "".join(source_lines)
-            entrypoint = script.__name__
-        else:
-            source = script
-            entrypoint = None
-        dedented = textwrap.dedent(source).strip()
-        if not dedented:
-            raise ValueError("RunScript code cannot be empty")
-        return dedented + "\n", entrypoint
-
-    @staticmethod
-    def _infer_entrypoint(script_source: str) -> str:
-        matches = re.findall(r"^def\s+([A-Za-z_][\w]*)\s*\(", script_source, re.MULTILINE)
-        if not matches:
-            raise ValueError(
-                "Could not determine the entrypoint for the RunScript code. Please set the `entrypoint` argument."
-            )
-        unique_matches = [match for match in matches if match]
-        if len(unique_matches) > 1:
-            raise ValueError(
-                "Multiple top-level functions were found in the RunScript code. Please specify the `entrypoint` argument."
-            )
-        return unique_matches[0]
-
     def is_source(self) -> bool:
         return len(self._inputs.inputs) == 0
 
@@ -544,7 +482,7 @@ def stage(
     outputs: dict[str, type[PipelineOutput]] | None = None,
     stage_name: str | None = None,
     params: dict[str, PipelineParameter | PipelineArgValueType] | None = None,
-) -> Callable[[Callable[..., dict[str, Any]]], RunScript]:
+) -> Callable[[Callable[..., dict[str, Any] | None]], RunScript]:
     """
     Decorator for building a RunScript stage from a Python function.
 
@@ -561,7 +499,7 @@ def stage(
     ...     return {"geometry": geometry}
     """
 
-    def decorator(fn: Callable[..., dict[str, Any]]) -> RunScript:
+    def decorator(fn: Callable[..., dict[str, Any] | None]) -> RunScript:
         return RunScript(
             script=fn,
             stage_name=stage_name,

luminarycloud/pipelines/stages.py

@@ -73,15 +73,15 @@ class ReadMesh(StandardStage[ReadMeshOutputs]):
     """
     Reads a Mesh into the Pipeline.
 
+    Does not complete until the Mesh is ready for simulation.  Fails if the meshing job completes
+    with errors.
+
     .. warning:: This feature is experimental and may change or be removed in the future.
 
     Parameters
     ----------
     mesh_id : str | StringPipelineParameter
         The ID of the Mesh to retrieve.
-    wait_timeout_seconds : int | IntPipelineParameter | None
-        The number of seconds to wait for the Mesh to be ready. If None, defaults to 1800 seconds
-        (30 minutes).
 
     Outputs
     -------
@@ -94,57 +94,60 @@ class ReadMesh(StandardStage[ReadMeshOutputs]):
         *,
         stage_name: str | None = None,
         mesh_id: str | StringPipelineParameter,
-        wait_timeout_seconds: int | IntPipelineParameter | None = None,
     ):
-        if wait_timeout_seconds is None:
-            wait_timeout_seconds = 30 * 60
         super().__init__(
             stage_name,
-            {"mesh_id": mesh_id, "wait_timeout_seconds": wait_timeout_seconds},
+            {"mesh_id": mesh_id},
             StageInputs(self),
             ReadMeshOutputs._instantiate_for(self),
         )
 
 
 @dataclass
-class ModifyGeometryOutputs(StageOutputs):
-    geometry: PipelineOutputGeometry
-    """The modified Geometry, represented as a new GeometryVersion."""
+class WaitForMeshOutputs(StageOutputs):
+    mesh: PipelineOutputMesh
+    """
+    The Mesh that will be waited for.
+    """
 
 
-# TODO: figure out what `mods` actually is. What does the non-pipeline geo mod interface look like?
-class ModifyGeometry(StandardStage[ModifyGeometryOutputs]):
+class WaitForMesh(StandardStage[WaitForMeshOutputs]):
     """
-    Modifies a Geometry.
+    Waits for a Mesh to be ready.
+
+    This is useful if you have a more complicated meshing setup than the `Mesh` stage can handle.
+
+    For example, you can use a `RunScript` stage to call `lc.create_or_get_mesh()` with arbitrary
+    meshing parameters, then pass the resulting mesh to this stage to wait for it to be ready.
+    Mesh creation can take minutes to hours, and `RunScript` has a short timeout, so you can't wait
+    for it to complete in the `RunScript` stage, but `lc.create_or_get_mesh()` returns immediately
+    and mesh creation happens asynchronously, so you can wait for it to be ready in this stage.
 
     .. warning:: This feature is experimental and may change or be removed in the future.
 
     Parameters
     ----------
-    mods : dict
-        The modifications to apply to the Geometry.
-    geometry : PipelineOutputGeometry
-        The Geometry to modify.
+    mesh : PipelineOutputMesh
+        The Mesh to wait for.
 
     Outputs
     -------
-    geometry : PipelineOutputGeometry
-        The modified Geometry, represented as a new GeometryVersion.
+    mesh : PipelineOutputMesh
+        The same Mesh that was passed in as input. It will now be in a `COMPLETED` status, and ready
+        to be used in a Simulation.
     """
 
     def __init__(
         self,
         *,
         stage_name: str | None = None,
-        mods: list[dict],
-        geometry: PipelineOutputGeometry,
+        mesh: PipelineOutputMesh,
     ):
-        raise NotImplementedError("ModifyGeometry is not implemented yet.")
         super().__init__(
             stage_name,
-            {"mods": mods},
-            StageInputs(self, geometry=(PipelineOutputGeometry, geometry)),
-            ModifyGeometryOutputs._instantiate_for(self),
+            {},
+            StageInputs(self, mesh=(PipelineOutputMesh, mesh)),
+            WaitForMeshOutputs._instantiate_for(self),
         )