luminarycloud 0.21.2__py3-none-any.whl → 0.22.1__py3-none-any.whl

luminarycloud/pipelines/stages.py

@@ -2,7 +2,7 @@
 from dataclasses import dataclass
 
 from .core import Stage, StageInputs, StageOutputs, PipelineOutput
-from .parameters import StringPipelineParameter, IntPipelineParameter
+from .parameters import BoolPipelineParameter, StringPipelineParameter, IntPipelineParameter
 from ..meshing import MeshGenerationParams
 
 
@@ -49,6 +49,20 @@ class ReadGeometry(Stage[ReadGeometryOutputs]):
     ----------
     geometry_id : str | StringPipelineParameter
         The ID of the Geometry to retrieve.
+    use_geo_without_copying : bool | BoolPipelineParameter
+        By default, this is False, meaning that each Geometry this stage references will be copied
+        and the PipelineJob will actually operate on the copied Geometry. This is done so that a
+        PipelineJob can be based on a single parametric Geometry which each PipelineJobRun modifies
+        by applying a NamedVariableSet. That modification mutates the Geometry, so those runs can
+        only happen in parallel without intefrering with each other if they each operate on a
+        different copy of the Geometry.
+
+        However, if you've already prepared your Geometry in advance and you don't want the
+        PipelineJob to create copies, you can set this to True. In that case, the referenced
+        Geometry will be used directly without being copied.
+
+        IMPORTANT: If you set this to True, you must ensure no two PipelineJobRuns operate on the
+        same Geometry, i.e. no two PipelineArgs rows contain the same Geometry ID.
 
     Outputs
     -------
@@ -61,10 +75,11 @@ class ReadGeometry(Stage[ReadGeometryOutputs]):
         *,
         stage_name: str | None = None,
         geometry_id: str | StringPipelineParameter,
+        use_geo_without_copying: bool | BoolPipelineParameter = False,
     ):
         super().__init__(
             stage_name,
-            {"geometry_id": geometry_id},
+            {"geometry_id": geometry_id, "use_geo_without_copying": use_geo_without_copying},
             StageInputs(self),
             ReadGeometryOutputs._instantiate_for(self),
         )
@@ -202,13 +217,6 @@ class Mesh(Stage[MeshOutputs]):
             MeshOutputs._instantiate_for(self),
         )
 
-    # TODO: bring back the full MeshGenerationParams, but we need to be able to hydrate it from the
-    # pipeline YAML. I can probably bake that logic into PipelineDictable, `from_pipeline_dict` or
-    # something.
-    # @classmethod
-    # def _parse_params(cls, params: dict) -> dict:
-    #     return {"mesh_gen_params": MeshGenerationParams.from_pipeline_dict(**params["mesh_gen_params"])}
-
 
 @dataclass
 class SimulateOutputs(StageOutputs):
@@ -230,6 +238,9 @@ class Simulate(Stage[SimulateOutputs]):
         The name to assign to the Simulation. If None, a default name will be used.
     sim_template_id : str | StringPipelineParameter
         The ID of the SimulationTemplate to use for the Simulation.
+    batch_processing : bool | BoolPipelineParameter, default True
+        If True, the Simulation will run as a standard job. If False, the Simulation will run as a
+        priority job.
 
     Outputs
     -------
@@ -244,10 +255,12 @@ class Simulate(Stage[SimulateOutputs]):
         mesh: PipelineOutputMesh,
         sim_name: str | StringPipelineParameter | None = None,
         sim_template_id: str | StringPipelineParameter,
+        batch_processing: bool | BoolPipelineParameter = True,
     ):
         super().__init__(
             stage_name,
             {
+                "batch_processing": batch_processing,
                 "sim_name": sim_name,
                 "sim_template_id": sim_template_id,
             },

luminarycloud/project.py

@@ -199,8 +199,6 @@ class Project(ProtoWrapperBase):
     def load_geometry_to_setup(self, geometry: "Geometry") -> None:
         """
         Load a geometry to the setup phase.
-        NOTE: this operation is irreversible and deletes all the existing meshes and simulations
-        in the project.
 
         Parameters
         ----------
@@ -487,15 +485,14 @@ class Project(ProtoWrapperBase):
         description : str, optional
             Simulation description.
         batch_processing : bool, default True
-            If True, batch processing will be used for this
-            simulation.
-            Use Batch Processing on simulations that are not time-sensitive to
-            save up to 65% in credits.
+            If True, this simulation will run as a standard job. If False, this simulation will run
+            as a priority job.
         gpu_type : GPUType, optional
             GPU type to use for the simulation.
         gpu_count : int, optional
-            Number of GPUs to use for the simulation. Must be specified to a
-            positive value if `gpu_type` is specified.
+            Number of GPUs to use for the simulation. Only relevant if `gpu_type` is
+            specified. If this is set to 0 or omitted and `gpu_type` is specified, the number
+            of gpus will be automatically determined.
         """
 
         named_variable_set_version_id: Optional[str] = None

luminarycloud/simulation.py

@@ -311,6 +311,19 @@ class Simulation(ProtoWrapperBase):
         req = simulationpb.GetSimulationParametersRequest(id=self.id)
         return SimulationParam.from_proto(get_default_client().GetSimulationParameters(req))
 
+    def _get_workflow_id(self) -> Optional[str]:
+        """
+        Retrieves the workflow ID associated with the current simulation.
+
+        Returns
+        -------
+        str | None
+            The workflow ID corresponding to this simulation's ID, or None if the simulation
+            has no associated workflow.
+        """
+        result = _get_workflow_ids([self.id])
+        return result.get(self.id)
+
     @deprecated(
         "Use get_parameters() instead. This method will be removed in a future release.",
     )
@@ -361,3 +374,47 @@ def get_simulation(id: SimulationID) -> Simulation:
     req = simulationpb.GetSimulationRequest(id=id)
     res = get_default_client().GetSimulation(req)
     return Simulation(res.simulation)
+
+
+def _get_workflow_ids(simulation_ids: list[SimulationID]) -> dict[SimulationID, str]:
+    """
+    Get the workflow IDs corresponding to simulation IDs.
+
+    This is useful for mapping between UI-created simulations (which have workflow IDs)
+    and the simulation IDs used in the API.
+
+    Parameters
+    ----------
+    simulation_ids : list[SimulationID]
+        The simulation IDs to look up.
+
+    Returns
+    -------
+    dict[SimulationID, str]
+        A mapping from simulation ID to workflow ID. Only simulation IDs that were
+        successfully resolved to workflow IDs are present as keys. Simulation IDs are
+        omitted if:
+
+        - The simulation ID does not exist
+        - The user lacks access to the simulation's project
+        - The simulation has no associated workflow ID
+
+    Examples
+    --------
+    >>> import luminarycloud as lc
+    >>> sim_ids = [lc.SimulationID("sim_123"), lc.SimulationID("sim_456")]
+    >>> workflow_ids = lc._get_workflow_ids(sim_ids)
+    >>> print(workflow_ids)
+    {SimulationID('sim_123'): 'wf_abc', SimulationID('sim_456'): 'wf_def'}
+
+    >>> # Check if a simulation has a workflow
+    >>> sim_id = lc.SimulationID("sim_123")
+    >>> if sim_id in workflow_ids:
+    ...     print(f"Workflow ID: {workflow_ids[sim_id]}")
+    ... else:
+    ...     print("No workflow found")
+    """
+    req = simulationpb.GetWorkflowIDsRequest(simulation_ids=simulation_ids)
+    res = get_default_client().GetWorkflowIDs(req)
+    # Return dict with SimulationID keys (not str keys)
+    return {SimulationID(sim_id): wf_id for sim_id, wf_id in res.data.items()}

luminarycloud/types/vector3.py

@@ -8,11 +8,10 @@ from .adfloat import (
     _to_ad_proto as _float_to_ad_proto,
     _from_ad_proto as _float_from_ad_proto,
 )
-from ..pipeline_util.dictable import PipelineDictable
 
 
 @dataclass
-class Vector3(PipelineDictable):
+class Vector3:
     """Represents a 3-dimensional vector.
 
     Supports direct component access, indexing, iteration, and conversion to numpy arrays.

luminarycloud/vis/data_extraction.py

@@ -602,7 +602,7 @@ class DataExtractor:
         code += f"    if sol.id == '{self._solution.id}':\n"
         code += f"        solution = sol\n"
         code += f"        break\n"
-        code += "data_extractor = vis.DataExtractor(solution)\n"
+        code += f"{obj_name} = vis.DataExtractor(solution)\n"
         code += "\n"
 
         code += "\n"
@@ -615,11 +615,11 @@ class DataExtractor:
         for extract in self._extracts:
             # Name objects numerically: slice0, slice1, etc.
             name = _data_extract_to_obj_name(extract)
-            obj_name = f"{name}{name_map[obj_name]}"
-            name_map[obj_name] += 1
-            ids_to_obj_name[extract.id] = obj_name
-            code += extract._to_code_helper(obj_name, hide_defaults=hide_defaults)
-            code += f"data_extractor.add_data_extract({obj_name})\n"
+            extract_obj_name = f"{name}{name_map[name]}"
+            name_map[name] += 1
+            ids_to_obj_name[extract.id] = extract_obj_name
+            code += extract._to_code_helper(extract_obj_name, hide_defaults=hide_defaults)
+            code += f"{obj_name}.add_data_extract({extract_obj_name})\n"
             code += "\n"
 
         if include_imports:
@@ -649,7 +649,7 @@ class DataExtractor:
         code = "\n".join(filtered_lines)
 
         code += "\n"
-        code += "extract_output = extractor.create_extracts(name='extract data', description='lonerg description')\n"
+        code += f"extract_output = {obj_name}.create_extracts(name='extract data', description='longer description')\n"
         code += "status = extract_output.wait()\n"
         code += "if status == ExtractStatusType.COMPLETED:\n"
         code += "    extract_output.save_files('data_extracts_prefix', True)\n"

luminarycloud/vis/interactive_report.py

@@ -1,6 +1,7 @@
 import io
+import numpy as np
 from .visualization import RenderOutput
-from .report import ReportEntry
+from .report import ReportEntry, ReportContext
 
 try:
     import luminarycloud_jupyter as lcj
@@ -8,6 +9,82 @@ except ImportError:
     lcj = None
 
 
+def _detect_outliers(
+    metadata: list[dict[str, str | float]],
+    output_fields: list[str],
+    percentile_threshold: float = 95.0,
+) -> list[int] | None:
+    """
+    Detect outliers using Mahalanobis distance.
+
+    Parameters
+    ----------
+    metadata : list[dict[str, str | float]]
+        List of metadata dictionaries for each row
+    output_fields : list[str]
+        List of output field names to use for outlier detection
+    percentile_threshold : float, optional
+        Percentile threshold for outlier detection (default: 95.0)
+
+    Returns
+    -------
+    list[int] | None
+        List of row indices that are outliers, or None if detection fails
+    """
+    # Need at least 2 fields for meaningful multivariate analysis
+    if len(output_fields) < 2:
+        return None
+
+    # Extract data for the specified output fields
+    try:
+        data = []
+        for row_metadata in metadata:
+            row_data = []
+            for field in output_fields:
+                value = row_metadata.get(field)
+                if value is None or isinstance(value, str):
+                    # Skip if field is missing or non-numeric
+                    return None
+                row_data.append(float(value))
+            data.append(row_data)
+
+        data_array = np.array(data)
+
+        # Need at least as many samples as dimensions for covariance matrix
+        if len(data_array) < len(output_fields):
+            return None
+
+        # Calculate mean and covariance matrix
+        mean_vec = np.mean(data_array, axis=0)
+        cov_matrix = np.cov(data_array.T)
+
+        # Check if covariance matrix is singular
+        if np.linalg.det(cov_matrix) == 0:
+            return None
+
+        # Invert covariance matrix
+        inv_cov_matrix = np.linalg.inv(cov_matrix)
+
+        # Calculate Mahalanobis distance for each point
+        distances = []
+        for point in data_array:
+            diff = point - mean_vec
+            distance = np.sqrt(diff @ inv_cov_matrix @ diff)
+            distances.append(distance)
+
+        # Determine outlier threshold using percentile
+        threshold = np.percentile(distances, percentile_threshold)
+
+        # Find outlier indices
+        outlier_indices = [i for i, d in enumerate(distances) if d > threshold]
+
+        return outlier_indices
+
+    except Exception:
+        # If anything goes wrong, return None (no outliers detected)
+        return None
+
+
 class InteractiveReport:
     """
     Interactive report widget with lazy loading for large datasets.
@@ -29,7 +106,7 @@ class InteractiveReport:
     # TODO Will/Matt: this list of report entries could be how we store stuff in the DB
     # for interactive reports, to reference the post proc. extracts. A report is essentially
     # a bunch of extracts + metadata.
-    def __init__(self, entries: list[ReportEntry]) -> None:
+    def __init__(self, entries: list[ReportEntry], context: ReportContext | None = None) -> None:
         if not lcj:
             raise ImportError("InteractiveScene requires luminarycloud[jupyter] to be installed")
 
@@ -37,6 +114,13 @@ class InteractiveReport:
         if len(self.entries) == 0:
             raise ValueError("Invalid number of entries, must be > 0")
 
+        # Validate and store context if provided
+        if context is not None:
+            self._validate_context(context)
+            self.context = context
+        else:
+            self.context = None
+
         # Determine grid dimensions by downloading first entry
         # to understand the structure (number of columns)
         first_entry = self.entries[0]
@@ -54,12 +138,55 @@ class InteractiveReport:
 
         nrows = len(self.entries)
 
+        # Prepare report context for the widget
+        context_dict = None
+        if self.context is not None:
+            context_dict = self.context.to_dict()
+
+            # Compute outlier indices if we have outputs
+            if self.context.outputs and len(self.context.outputs) >= 2:
+                outlier_indices = _detect_outliers(
+                    [re._metadata for re in self.entries], self.context.outputs
+                )
+                if outlier_indices is not None:
+                    context_dict["outlier_indices"] = outlier_indices
+
         # Create widget with metadata but without data
-        self.widget = lcj.EnsembleWidget([re._metadata for re in self.entries], nrows, ncols)
+        self.widget = lcj.EnsembleWidget(
+            [re._metadata for re in self.entries], nrows, ncols, report_context=context_dict
+        )
 
         # Set the callback for lazy loading row data
         self.widget.set_row_data_callback(self._load_row_data)
 
+    def _validate_context(self, context: ReportContext) -> None:
+        """
+        Validate that all inputs and outputs from the ReportContext exist in the
+        first report entry's metadata.
+
+        Raises:
+        -------
+        ValueError
+            If any inputs or outputs are missing from the first entry's metadata.
+        """
+        first_entry = self.entries[0]
+        metadata_keys = set(first_entry._metadata.keys())
+
+        # Check for missing inputs
+        missing_inputs = [key for key in context.inputs if key not in metadata_keys]
+
+        # Check for missing outputs
+        missing_outputs = [key for key in context.outputs if key not in metadata_keys]
+
+        # Raise exception if any keys are missing
+        if missing_inputs or missing_outputs:
+            error_parts = []
+            if missing_inputs:
+                error_parts.append(f"Missing inputs: {missing_inputs}")
+            if missing_outputs:
+                error_parts.append(f"Missing outputs: {missing_outputs}")
+            raise ValueError(f"ReportContext validation failed. {', '.join(error_parts)}")
+
     def _load_row_data(self, row: int) -> None:
         """
         Load and send data for a specific row to the widget.
@@ -79,11 +206,29 @@ class InteractiveReport:
                 image_and_label = extract.download_images()
                 # Each image gets its own column
                 for il in image_and_label:
-                    self.widget.set_cell_data(row, col, il[0].getvalue(), "jpg")
+                    # il is a tuple of (BytesIO, label)
+                    # Use camera label for the name, fallback to "image" if empty
+                    camera_label = il[1]
+                    name = camera_label if camera_label else "image"
+                    # For description: prefer extract.description, then camera label, then fallback message
+                    description = (
+                        extract.description
+                        if extract.description
+                        else camera_label if camera_label else "no label or description provided"
+                    )
+                    self.widget.set_cell_data(
+                        row,
+                        col,
+                        il[0].getvalue(),
+                        "jpg",
+                        name=name,
+                        description=description,
+                    )
                     col += 1
             else:
                 plot_data = extract.download_data()
-                data = plot_data[0][0]
+                data = plot_data[0][0]  # The CSV data (rows)
+                plot_label = plot_data[0][1]  # The label from the extract
                 all_axis_labels = data[0]
 
                 axis_data = []
@@ -91,14 +236,25 @@ class InteractiveReport:
                     axis_values = [row[axis_idx] for row in data[1:]]
                     axis_data.append(axis_values)
 
+                # For plots: use extract.name, then plot_label, then "plot" as fallback
+                # For description: use extract.description, fallback to message if empty
+                name = extract.name if extract.name else (plot_label if plot_label else "plot")
+                description = (
+                    extract.description
+                    if extract.description
+                    else "no label or description provided"
+                )
+
                 self.widget.set_cell_scatter_plot(
                     row,
                     col,
-                    f"Row #{row} - Multi-axis Plot",
+                    name,  # Use the same name for the plot title
                     all_axis_labels,
                     axis_data,
-                    plot_name=f"plot-{row}",
+                    plot_name=name,
                     plot_mode="markers",
+                    name=name,
+                    description=description,
                 )
                 col += 1
 

luminarycloud/vis/report.py

@@ -1,3 +1,4 @@
+import csv as csv_module
 import json
 import os
 from typing import TYPE_CHECKING
@@ -17,6 +18,79 @@ if TYPE_CHECKING:
     from .interactive_report import InteractiveReport
 
 
+class ReportContext:
+    """
+    Context for interactive reports that defines input and output metadata keys.
+    Inputs define what the geometric and flow conditions are varied with running
+    data generation and the outputs define what quantities are extracted from
+    the simulations. For the report context to be valid we require that the both
+    the inputs and outputs are non-empty.
+
+    Attributes:
+    -----------
+    inputs : list[str]
+        List of metadata keys (column names) that represent inputs to the report.
+    outputs : list[str]
+        List of metadata keys (column names) that represent outputs from the report.
+    """
+
+    def __init__(self, inputs: list[str], outputs: list[str]) -> None:
+        self.inputs = inputs
+        self.outputs = outputs
+
+    def to_dict(self) -> dict:
+        """Convert ReportContext to a dictionary for serialization."""
+        return {
+            "inputs": self.inputs,
+            "outputs": self.outputs,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict) -> "ReportContext":
+        """Create a ReportContext from a dictionary.
+
+        Parameters:
+        -----------
+        data : dict
+            Dictionary containing 'inputs' and 'outputs' keys.
+
+        Raises:
+        -------
+        ValueError
+            If 'inputs' or 'outputs' keys are missing from the data.
+        """
+        if "inputs" not in data:
+            raise ValueError("ReportContext.from_dict: missing required key 'inputs'")
+        if "outputs" not in data:
+            raise ValueError("ReportContext.from_dict: missing required key 'outputs'")
+
+        inputs = data["inputs"]
+        outputs = data["outputs"]
+
+        if not isinstance(inputs, list):
+            raise ValueError(
+                f"ReportContext.from_dict: 'inputs' must be a list, got {type(inputs).__name__}"
+            )
+        if not isinstance(outputs, list):
+            raise ValueError(
+                f"ReportContext.from_dict: 'outputs' must be a list, got {type(outputs).__name__}"
+            )
+
+        if len(inputs) == 0:
+            raise ValueError("ReportContext.from_dict: 'inputs' must be non-empty")
+        if len(outputs) == 0:
+            raise ValueError("ReportContext.from_dict: 'outputs' must be non-empty")
+
+        return cls(inputs=inputs, outputs=outputs)
+
+
+def load_report_context_from_json(filepath: str) -> ReportContext:
+    """Load a ReportContext object from a JSON file at the given file path."""
+    with open(filepath, "r") as f:
+        data = json.load(f)
+    return ReportContext.from_dict(data)
+
+
 # TODO Will/Matt: this could be something like what we store in the DB
 # A report can contain a list of report entries that reference post proc.
 # extracts + styling info for how they should be displayed
@@ -86,7 +160,7 @@ class ReportEntry:
                 if res.HasField("line_data")
                 else RenderOutput(_InternalToken())
             )
-            extract._set_data(eid, self._project_id, eid, eid, status)
+            extract._set_data(eid, self._project_id, "", "", status)
             self._extracts.append(extract)
 
 
@@ -250,3 +324,41 @@ def load_report_from_json(filepath: str) -> "Report":
     with open(filepath, "r") as f:
         data = json.load(f)
     return Report.from_dict(data)
+
+
+def load_report_from_csv(filepath: str) -> "Report":
+    """Load a Report object from a CSV file at the given file path.
+
+    Each row in the CSV corresponds to a ReportEntry. Each column is converted
+    to metadata. No extracts are created when loading from CSV.
+
+    Parameters
+    ----------
+    filepath : str
+        Path to the CSV file to load.
+
+    Returns
+    -------
+    Report
+        A Report object with entries populated from the CSV rows.
+    """
+    entries = []
+    with open(filepath, "r") as f:
+        reader = csv_module.DictReader(f)
+        for row in reader:
+            # Convert all columns to metadata
+            metadata: dict[str, str | float] = {}
+            for key, value in row.items():
+                # Try to convert to float, otherwise keep as string
+                try:
+                    metadata[key] = float(value)
+                except (ValueError, TypeError):
+                    metadata[key] = value
+
+            # Create ReportEntry with placeholder project_id and no extracts
+            # We only need the project id for loading extracts, so we can omit it for CSV
+            # imports.
+            entry = ReportEntry(project_id="p-placeholder", extract_ids=[], metadata=metadata)
+            entries.append(entry)
+
+    return Report(entries)