luminarycloud 0.16.0__py3-none-any.whl → 0.16.2__py3-none-any.whl

luminarycloud/pipelines/core.py

@@ -1,14 +1,27 @@
 # Copyright 2025 Luminary Cloud, Inc. All Rights Reserved.
 from abc import ABC, abstractmethod
 from dataclasses import is_dataclass, fields
-from typing import Type, TypeVar, Generic
+from typing import Any, Type, TypeVar, Generic
+from typing_extensions import Self
 import re
 import yaml
 
-from .._helpers.warnings import experimental
 from ..pipeline_util.yaml import ensure_yamlizable
 
 
+class PipelineParameterRegistry:
+    def __init__(self):
+        self.parameters = {}
+
+    def register(self, parameter_class: Type["PipelineParameter"]) -> None:
+        self.parameters[parameter_class._type_name()] = parameter_class
+
+    def get(self, type_name: str) -> Type["PipelineParameter"]:
+        if type_name not in self.parameters:
+            raise ValueError(f"Unknown parameter type: {type_name}")
+        return self.parameters[type_name]
+
+
 class PipelineParameter(ABC):
     """
     Base class for all concrete PipelineParameters.
@@ -20,10 +33,16 @@ class PipelineParameter(ABC):
 
     @property
     def type(self) -> str:
-        return self._type()
+        return self.__class__._type_name()
+
+    @classmethod
+    @abstractmethod
+    def _represented_type(cls) -> Type:
+        pass
 
+    @classmethod
     @abstractmethod
-    def _type(self) -> str:
+    def _type_name(cls) -> str:
         pass
 
     def _validate(self) -> None:
@@ -42,6 +61,30 @@ class PipelineParameter(ABC):
     def _to_pipeline_dict(self) -> tuple[dict, list["PipelineParameter"]]:
         return {"$pipeline_param": self.name}, [self]
 
+    def __str__(self) -> str:
+        return f'{self.__class__.__name__}(name="{self.name}")'
+
+    _registry = PipelineParameterRegistry()
+
+    def __init_subclass__(cls, **kwargs):
+        super().__init_subclass__(**kwargs)
+        PipelineParameter._registry.register(cls)
+
+    @classmethod
+    def _get_subclass(cls, parameter_type: str) -> Type["PipelineParameter"]:
+        return cls._registry.get(parameter_type)
+
+    def _is_valid_value(self, value: Any) -> bool:
+        return isinstance(value, self._represented_type())
+
+    def __hash__(self) -> int:
+        return hash((self.type, self.name))
+
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, PipelineParameter):
+            return False
+        return self.__hash__() == other.__hash__()
+
 
 class PipelineInput:
     """
@@ -138,6 +181,25 @@ class OperatorOutputs(ABC):
             outputs[field.name] = field.type(owner, field.name)
         return cls(**outputs)
 
+    def downstream_inputs(self) -> list[PipelineInput]:
+        inputs = []
+        for field in fields(self):
+            inputs.extend(getattr(self, field.name).downstream_inputs)
+        return inputs
+
+
+class OperatorRegistry:
+    def __init__(self):
+        self.operators = {}
+
+    def register(self, operator_class: Type["Operator"]) -> None:
+        self.operators[operator_class.__name__] = operator_class
+
+    def get(self, operator_name: str) -> Type["Operator"]:
+        if operator_name not in self.operators:
+            raise ValueError(f"Unknown operator: {operator_name}")
+        return self.operators[operator_name]
+
 
 TOutputs = TypeVar("TOutputs", bound=OperatorOutputs)
 
@@ -157,53 +219,92 @@ class Operator(Generic[TOutputs], ABC):
         self.outputs = outputs
         ensure_yamlizable(self._params_dict()[0], "Operator parameters")
 
-    def _to_dict(self, id_for_task: dict) -> tuple[dict, list[PipelineParameter]]:
-        params, params_list = self._params_dict()
+    def is_source(self) -> bool:
+        return len(self._inputs.inputs) == 0
+
+    def inputs_dict(self) -> dict[str, tuple["Operator", str]]:
+        inputs = {}
+        for pipeline_input in self._inputs.inputs:
+            inputs[pipeline_input.name] = (
+                pipeline_input.upstream_output.owner,
+                pipeline_input.upstream_output.name,
+            )
+        return inputs
+
+    def downstream_tasks(self) -> list["Operator"]:
+        return [input.owner for input in self.outputs.downstream_inputs()]
+
+    def _to_dict(self, id_for_task: dict) -> tuple[dict, set[PipelineParameter]]:
+        params, pipeline_params_set = self._params_dict()
         d = {
             "name": self._task_name,
             "operator": self._operator_name,
             "params": params,
             "inputs": self._inputs._to_dict(id_for_task),
         }
-        return d, params_list
+        return d, pipeline_params_set
 
-    def _params_dict(self) -> tuple[dict, list[PipelineParameter]]:
+    def _params_dict(self) -> tuple[dict, set[PipelineParameter]]:
         d = {}
-        params = []
+        pipeline_params = set()
         for name, value in self._params.items():
             if hasattr(value, "_to_pipeline_dict"):
                 d[name], downstream_params = value._to_pipeline_dict()
-                params.extend(downstream_params)
+                pipeline_params.update(downstream_params)
             else:
                 d[name] = value
-        return d, params
+        return d, pipeline_params
 
     def __str__(self) -> str:
         return f'{self._operator_name}(name="{self._task_name}")'
 
+    _registry = OperatorRegistry()
+
+    def __init_subclass__(cls, **kwargs):
+        super().__init_subclass__(**kwargs)
+        Operator._registry.register(cls)
+
+    @classmethod
+    def _get_subclass(cls, operator_name: str) -> Type["Operator"]:
+        return cls._registry.get(operator_name)
+
+    @classmethod
+    def _parse_params(cls, params: dict) -> dict:
+        # Operators with params that are just primitives or PipelineParams have no parsing to do.
+        # Operators with more complicated params should override this method.
+        return params
+
 
-@experimental
 class Pipeline:
-    def __init__(self, name: str, tasks: list[Operator]):
-        self.name = name
+    def __init__(self, tasks: list[Operator]):
         self.tasks = tasks
+        self._task_ids = self._assign_ids_to_tasks()
 
     def to_yaml(self) -> str:
         return yaml.safe_dump(self._to_dict())
 
-    def _to_dict(self) -> dict:
-        id_for_task = self._assign_ids_to_tasks()
+    def pipeline_params(self) -> set[PipelineParameter]:
+        return self._tasks_dict_and_params()[1]
+
+    def _get_task_id(self, task: Operator) -> str:
+        return self._task_ids[task]
+
+    def _tasks_dict_and_params(self) -> tuple[dict, set[PipelineParameter]]:
+        id_for_task = self._task_ids
         tasks = {}
-        params = []
+        params = set()
         for task in id_for_task.keys():
             task_dict, referenced_params = task._to_dict(id_for_task)
             tasks[id_for_task[task]] = task_dict
-            params.extend(referenced_params)
+            params.update(referenced_params)
+        return tasks, params
+
+    def _to_dict(self) -> dict:
+        tasks, params = self._tasks_dict_and_params()
 
         d = {
             "lc_pipeline": {
                 "schema_version": 1,
-                "name": self.name,
                 "params": self._pipeline_params_dict(params),
                 "tasks": tasks,
             }
@@ -214,7 +315,7 @@ class Pipeline:
     def _assign_ids_to_tasks(self) -> dict[Operator, str]:
         return {task: f"t{i + 1}-{task._operator_name}" for i, task in enumerate(self.tasks)}
 
-    def _pipeline_params_dict(self, params: list[PipelineParameter]) -> dict:
+    def _pipeline_params_dict(self, params: set[PipelineParameter]) -> dict:
         d: dict[str, dict] = {}
         for p in params:
             if p.name in d and d[p.name]["type"] != p.type:
@@ -223,3 +324,86 @@ class Pipeline:
                 )
             d[p.name] = {"type": p.type}
         return d
+
+    @classmethod
+    def _from_yaml(cls, yaml_str: str) -> Self:
+        d = yaml.safe_load(yaml_str)
+        if "lc_pipeline" not in d:
+            raise ValueError("Invalid pipeline YAML: missing 'lc_pipeline' key")
+
+        d = d["lc_pipeline"]
+        if "schema_version" not in d:
+            raise ValueError("Invalid pipeline YAML: missing 'schema_version' key")
+        if "tasks" not in d:
+            raise ValueError("Invalid pipeline YAML: missing 'tasks' key")
+
+        if d["schema_version"] != 1:
+            raise ValueError(f"Unsupported schema version: {d['schema_version']}")
+
+        # first, parse the pipeline parameters...
+        parsed_params = {}
+        for param_name, param_metadata in d.get("params", {}).items():
+            parsed_params[param_name] = PipelineParameter._get_subclass(param_metadata["type"])(
+                param_name
+            )
+
+        # ...and use them as replacements for any references in the tasks' parameters
+        for task_dict in d["tasks"].values():
+            task_dict["params"] = _recursive_replace_pipeline_params(
+                task_dict["params"], parsed_params
+            )
+
+        # then, finish parsing the tasks
+        parsed_tasks = {}
+        for task_id in d["tasks"]:
+            _parse_task(d, task_id, parsed_tasks)
+
+        return cls(list(parsed_tasks.values()))
+
+
+def _recursive_replace_pipeline_params(d: Any, parsed_params: dict) -> Any:
+    if isinstance(d, dict):
+        if "$pipeline_param" in d:
+            # d is a dict representation of a PipelineParameter, so return the actual PipelineParameter
+            pp_name = d["$pipeline_param"]
+            if pp_name not in parsed_params:
+                raise ValueError(
+                    f'Pipeline parameter "{pp_name}" referenced in a pipeline task, but not found in pipeline\'s declared parameters'
+                )
+            return parsed_params[pp_name]
+        else:
+            return {
+                key: _recursive_replace_pipeline_params(value, parsed_params)
+                for key, value in d.items()
+            }
+    elif isinstance(d, list):
+        return [_recursive_replace_pipeline_params(item, parsed_params) for item in d]
+    else:
+        return d
+
+
+def _parse_task(pipeline_dict: dict, task_id: str, all_tasks: dict[str, Operator]) -> Operator:
+    all_tasks_dict = pipeline_dict["tasks"]
+    if task_id in all_tasks:
+        return all_tasks[task_id]
+    task_dict = all_tasks_dict[task_id]
+    operator_name = task_dict["operator"]
+    operator_class = Operator._get_subclass(operator_name)
+
+    parsed_inputs = {}
+    for input_name, input_value in task_dict["inputs"].items():
+        source_task_id, source_output_name = input_value.split(".")
+        source_task = _parse_task(pipeline_dict, source_task_id, all_tasks)
+        source_output = getattr(source_task.outputs, source_output_name)
+        parsed_inputs[input_name] = source_output
+
+    parsed_params = operator_class._parse_params(task_dict["params"])
+
+    op_params = {
+        "task_name": task_dict["name"],
+        **parsed_params,
+        **parsed_inputs,
+    }
+    operator = operator_class(**op_params)
+    all_tasks[task_id] = operator
+    return operator

luminarycloud/pipelines/operators.py

@@ -1,7 +1,6 @@
 # Copyright 2025 Luminary Cloud, Inc. All Rights Reserved.
 from dataclasses import dataclass
 
-from .._helpers.warnings import experimental
 from .core import Operator, OperatorInputs, OperatorOutputs, PipelineOutput
 from .parameters import StringPipelineParameter
 from ..meshing import MeshGenerationParams
@@ -40,7 +39,6 @@ class ReadGeometryOutputs(OperatorOutputs):
     """
 
 
-@experimental
 class ReadGeometry(Operator[ReadGeometryOutputs]):
     """
     Reads a Geometry into the Pipeline.
@@ -79,7 +77,6 @@ class ModifyGeometryOutputs(OperatorOutputs):
 
 
 # TODO: figure out what `mods` actually is. What does the non-pipeline geo mod interface look like?
-@experimental
 class ModifyGeometry(Operator[ModifyGeometryOutputs]):
     """
     Modifies a Geometry.
@@ -121,15 +118,14 @@ class MeshOutputs(OperatorOutputs):
     """The Mesh generated from the given Geometry."""
 
 
-@experimental
 class Mesh(Operator[MeshOutputs]):
     """
     Generates a Mesh from a Geometry.
 
     Parameters
     ----------
-    mesh_gen_params : MeshGenerationParams
-        The parameters to use for mesh generation.
+    max_cv_count : int
+        The maximum number of control volumes to generate.
     geometry : PipelineOutputGeometry
         The Geometry to mesh.
 
@@ -145,16 +141,23 @@ class Mesh(Operator[MeshOutputs]):
         self,
         *,
         task_name: str | None = None,
-        mesh_gen_params: MeshGenerationParams,
+        max_cv_count: int,
         geometry: PipelineOutputGeometry,
     ):
         super().__init__(
             task_name,
-            {"mesh_gen_params": mesh_gen_params},
+            {"max_cv_count": max_cv_count},
             OperatorInputs(self, geometry=(PipelineOutputGeometry, geometry)),
             MeshOutputs._instantiate_for(self),
         )
 
+    # TODO: bring back the full MeshGenerationParams, but we need to be able to hydrate it from the
+    # pipeline YAML. I can probably bake that logic into PipelineDictable, `from_pipeline_dict` or
+    # something.
+    # @classmethod
+    # def _parse_params(cls, params: dict) -> dict:
+    #     return {"mesh_gen_params": MeshGenerationParams.from_pipeline_dict(**params["mesh_gen_params"])}
+
 
 @dataclass
 class SimulateOutputs(OperatorOutputs):
@@ -162,7 +165,6 @@ class SimulateOutputs(OperatorOutputs):
     """The Simulation."""
 
 
-@experimental
 class Simulate(Operator[SimulateOutputs]):
     """
     Runs a Simulation.

luminarycloud/pipelines/parameters.py

@@ -1,5 +1,6 @@
 # Copyright 2025 Luminary Cloud, Inc. All Rights Reserved.
 from .core import PipelineParameter
+from typing import Type
 
 
 class StringPipelineParameter(PipelineParameter):
@@ -8,7 +9,12 @@ class StringPipelineParameter(PipelineParameter):
     allow its value to be set when the Pipeline is invoked.
     """
 
-    def _type(self) -> str:
+    @classmethod
+    def _represented_type(cls) -> Type:
+        return str
+
+    @classmethod
+    def _type_name(cls) -> str:
         return "string"
 
 
@@ -18,7 +24,12 @@ class FloatPipelineParameter(PipelineParameter):
     allow its value to be set when the Pipeline is invoked.
     """
 
-    def _type(self) -> str:
+    @classmethod
+    def _represented_type(cls) -> Type:
+        return float
+
+    @classmethod
+    def _type_name(cls) -> str:
         return "float"
 
 
@@ -28,7 +39,12 @@ class IntPipelineParameter(PipelineParameter):
     allow its value to be set when the Pipeline is invoked.
     """
 
-    def _type(self) -> str:
+    @classmethod
+    def _represented_type(cls) -> Type:
+        return int
+
+    @classmethod
+    def _type_name(cls) -> str:
         return "int"
 
 
@@ -38,5 +54,10 @@ class BoolPipelineParameter(PipelineParameter):
     allow its value to be set when the Pipeline is invoked.
     """
 
-    def _type(self) -> str:
+    @classmethod
+    def _represented_type(cls) -> Type:
+        return bool
+
+    @classmethod
+    def _type_name(cls) -> str:
         return "bool"

luminarycloud/project.py

@@ -27,7 +27,7 @@ from ._helpers import (
     upload_mesh,
     upload_table_as_json,
 )
-from ._helpers.warnings import deprecated, experimental
+from ._helpers.warnings import deprecated
 from ._proto.api.v0.luminarycloud.geometry import geometry_pb2 as geometrypb
 from ._proto.api.v0.luminarycloud.mesh import mesh_pb2 as meshpb
 from ._proto.api.v0.luminarycloud.named_variable_set import (
@@ -39,6 +39,7 @@ from ._proto.api.v0.luminarycloud.simulation_template import (
     simulation_template_pb2 as simtemplatepb,
 )
 from ._proto.client import simulation_pb2 as clientpb
+from ._proto.table import table_pb2 as tablepb
 from ._proto.hexmesh import hexmesh_pb2 as hexmeshpb
 from ._proto.upload import upload_pb2 as uploadpb
 from ._wrapper import ProtoWrapper, ProtoWrapperBase
@@ -440,15 +441,15 @@ class Project(ProtoWrapperBase):
             raise RuntimeError("The table upload failed.")
 
         # Update the simulation template with the new table reference.
-        params: clientpb.SimulationParam = simulation_template._proto.parameters
-        params.table_references[name].url = url
-        params.table_references[name].table_type = table_type.value
-        params.table_references[name].uploaded_filename = uploaded_filename
-        simulation_template._update(name=simulation_template.name, parameters=params)
+        params: SimulationParam = simulation_template.get_parameters()
+        params._table_references[name] = tablepb.Metadata()
+        params._table_references[name].url = url
+        params._table_references[name].table_type = table_type.value
+        params._table_references[name].uploaded_filename = uploaded_filename
+        simulation_template.update(parameters=params)
+        # The name is lost in to/from proto conversions so make it equal to the id for consistency.
+        return RectilinearTable(id=name, name=name, table_type=table_type)
 
-        return RectilinearTable(id=name, name=uploaded_filename, table_type=table_type)
-
-    @experimental
     def set_surface_deformation(
         self,
         file_path: PathLike | str,
@@ -608,14 +609,13 @@ class Project(ProtoWrapperBase):
         res = get_default_client().CreateSimulationTemplate(req)
         return lc.SimulationTemplate(res.simulation_template)
 
-    @experimental
     def create_named_variable_set(
         self, name: str, named_variables: dict[str, LcFloat]
     ) -> NamedVariableSet:
         """
         Create a new named variable set.
 
-        Note: This feature is experimental and may change or be removed without notice.
+        .. warning:: This feature is experimental and may change or be removed without notice.
         """
         req = namedvariablepb.CreateNamedVariableSetRequest(
             project_id=self.id,
@@ -636,7 +636,6 @@ class Project(ProtoWrapperBase):
         return [lc.NamedVariableSet(n) for n in res.named_variable_sets]
 
 
-@experimental
 def add_named_variables_from_csv(project: Project, csv_path: str) -> list[NamedVariableSet]:
     """
     This function reads named variables from a CSV file and creates corresponding NamedVariableSets in the given project.
@@ -644,6 +643,8 @@ def add_named_variables_from_csv(project: Project, csv_path: str) -> list[NamedV
     name, var1, var2, ...
     name1, val1, val2, ...
     name2, val1, val2, ...
+
+    .. warning:: This feature is experimental and may change or be removed without notice.
     """
     import csv
 

luminarycloud/simulation_param.py

@@ -6,12 +6,10 @@ from os import PathLike
 from pprint import pformat
 from typing import Optional, TypeVar, cast
 
-import luminarycloud.params.enum._enum_wrappers as enum
 from luminarycloud._helpers._simulation_params_from_json import (
     simulation_params_from_json_path,
     simulation_params_from_json_dict,
 )
-from luminarycloud._helpers.warnings import experimental
 from luminarycloud._proto.client import simulation_pb2 as clientpb
 from luminarycloud._proto.client.entity_pb2 import EntityIdentifier
 from luminarycloud._proto.output import output_pb2 as outputpb
@@ -38,7 +36,7 @@ from luminarycloud.params.simulation.volume_entity_ import (
 from luminarycloud.params.simulation import (
     SimulationParam as _SimulationParam,
 )
-from luminarycloud.reference_values import ReferenceValues, ReferenceValuesType
+from luminarycloud.reference_values import ReferenceValues
 from luminarycloud.types import Vector3Like
 from luminarycloud.types.vector3 import _to_vector3_ad_proto
 
@@ -152,7 +150,10 @@ class SimulationParam(_SimulationParam):
             _list=volume_material_pairs,
             _accessor=lambda v: get_id(v.volume_identifier),
             _to_remove=volume_identifier.id,
-            _warning_message=lambda v: f"Volume {_stringify_identifier(volume_identifier)} has already been assigned material {_stringify_identifier(v.material_identifier)}. Overwriting...",
+            _warning_message=lambda v: (
+                f"Volume {_stringify_identifier(volume_identifier)} has already been assigned "
+                f"material {_stringify_identifier(v.material_identifier)}. Overwriting..."
+            ),
         )
 
         if volume_identifier.id not in (get_id(v.volume_identifier) for v in self.volume_entity):
@@ -226,13 +227,19 @@ class SimulationParam(_SimulationParam):
             _list=volume_physics_pairs,
             _accessor=lambda v: get_id(v.volume_identifier),
             _to_remove=volume_identifier.id,
-            _warning_message=lambda v: f"Volume {_stringify_identifier(volume_identifier)} has already been assigned physics {_stringify_identifier(v.physics_identifier)}. Overwriting...",
+            _warning_message=lambda v: (
+                f"Volume {_stringify_identifier(volume_identifier)} has already been assigned "
+                f"physics {_stringify_identifier(v.physics_identifier)}. Overwriting..."
+            ),
         )
         _remove_from_list_with_warning(
             _list=volume_physics_pairs,
             _accessor=lambda v: get_id(v.physics_identifier),
             _to_remove=get_id(physics.physics_identifier),
-            _warning_message=lambda v: f"Physics {_stringify_identifier(physics.physics_identifier)} has already been assigned to volume {_stringify_identifier(v.volume_identifier)}. Overwriting...",
+            _warning_message=lambda v: (
+                f"Physics {_stringify_identifier(physics.physics_identifier)} has already been "
+                f"assigned to volume {_stringify_identifier(v.volume_identifier)}. Overwriting..."
+            ),
         )
 
         if volume_identifier.id not in (get_id(v.volume_identifier) for v in self.volume_entity):
@@ -249,7 +256,6 @@ class SimulationParam(_SimulationParam):
             )
         )
 
-    @experimental
     def configure_adjoint_surface_output(
         self,
         quantity_type: QuantityType,
@@ -268,25 +274,25 @@ class SimulationParam(_SimulationParam):
         .. warning:: This feature is experimental and may change or be removed without notice.
         """
         self.adjoint = self.adjoint or Adjoint()
-        self.adjoint.output = outputpb.Output()
-        self.adjoint.output.quantity = quantity_type.value
-        self.adjoint.output.in_surfaces.extend(surface_ids)
-        self.adjoint.output.frame_id = frame_id
+        self.adjoint._output = outputpb.Output()
+        self.adjoint._output.quantity = quantity_type.value
+        self.adjoint._output.in_surfaces.extend(surface_ids)
+        self.adjoint._output.frame_id = frame_id
         if QuantityType._is_average(quantity_type):
             if averaging_type == AveragingType.UNSPECIFIED:
-                self.adjoint.output.surface_average_properties.averaging_type = (
+                self.adjoint._output.surface_average_properties.averaging_type = (
                     SpaceAveragingType.NO_AVERAGING.value
                 )
             elif averaging_type == AveragingType.AREA:
-                self.adjoint.output.surface_average_properties.averaging_type = (
+                self.adjoint._output.surface_average_properties.averaging_type = (
                     SpaceAveragingType.AREA.value
                 )
             elif averaging_type == AveragingType.MASS_FLOW:
-                self.adjoint.output.surface_average_properties.averaging_type = (
+                self.adjoint._output.surface_average_properties.averaging_type = (
                     SpaceAveragingType.MASS_FLOW.value
                 )
         elif QuantityType._is_force(quantity_type):
-            self.adjoint.output.force_properties.CopyFrom(
+            self.adjoint._output.force_properties.CopyFrom(
                 outputpb.ForceProperties(
                     force_dir_type=(
                         outputpb.FORCE_DIRECTION_BODY_ORIENTATION_AND_FLOW_DIR
@@ -332,9 +338,15 @@ from luminarycloud import outputs, SimulationParam, EntityIdentifier
 
 
 """
-        return code + self._to_code_helper("obj", hide_defaults, use_tmp_objs).replace(
-            "luminarycloud.simulation_param.SimulationParam()", "SimulationParam()", 1
+        code += self._to_code_helper("obj", hide_defaults, use_tmp_objs).replace(
+            "luminarycloud.simulation_param.SimulationParam()", "luminarycloud.SimulationParam()", 1
         )
+        if self.adjoint.primal_simulation_id != "":
+            code += """
+# TODO(USER): Select appropriate parameters to configure the adjoint output.
+obj.configure_adjoint_output("...")
+"""
+        return code
 
     def find_parameter(self, parameter: str) -> None:
         """