simstack 2.0.1__py3-none-any.whl

simstack/models/images2d.py

@@ -0,0 +1,113 @@
+from typing import Optional, Literal, Dict, Any, Union
+import base64
+import os
+from odmantic import Model, Field, ObjectId
+from simstack.models.simstack_model import simstack_model
+
+@simstack_model
+class Image2DArtifactModel(Model):
+    """Model for storing 2D image artifacts in MongoDB."""
+
+    parent_id: Optional[ObjectId] = None
+    name: str = Field(..., description="Name of the image artifact")
+    description: Optional[str] = Field(None, description="Description of the image")
+    format: Literal["png", "jpg", "jpeg", "svg", "gif", "bmp", "webp"] = Field(
+        ..., description="Image format"
+    )
+    data: bytes = Field(..., description="Binary image data")
+    width: Optional[int] = Field(None, description="Image width in pixels")
+    height: Optional[int] = Field(None, description="Image height in pixels")
+    metadata: Dict[str, Any] = Field(default_factory=dict, description="Additional metadata")
+
+    def to_base64(self) -> str:
+        """Convert binary data to base64 string."""
+        return base64.b64encode(self.data).decode("utf-8")
+
+    def get_data_uri(self) -> str:
+        """Get data URI for the image."""
+        mime_type = f"image/{self.format}"
+        if self.format == "svg":
+            mime_type = "image/svg+xml"
+        return f"data:{mime_type};base64,{self.to_base64()}"
+
+    def make_table_entries(
+        self,
+        max_recursion_level=1,
+        drop_id=True,
+        current_level=0,
+        visited=None,
+        field_prefix="",
+    ):
+        return {
+            "name": self.name,
+            "format": self.format,
+            "size": f"{self.width}x{self.height}" if self.width and self.height else "unknown"
+        }
+
+    def make_column_defs_instance(
+        self,
+        table_name=None,
+        max_recursion_level=1,
+        drop_id=True,
+        current_level=0,
+        visited=None,
+        field_prefix="",
+    ):
+        return [
+            {"field": "name", "headerName": "Name"},
+            {"field": "format", "headerName": "Format"},
+            {"field": "size", "headerName": "Size"},
+        ]
+
+def create_image_artifact(
+    name: str,
+    data: bytes,
+    format: str,
+    description: Optional[str] = None,
+    parent_id: Optional[ObjectId] = None,
+    width: Optional[int] = None,
+    height: Optional[int] = None,
+    metadata: Optional[Dict[str, Any]] = None,
+) -> Image2DArtifactModel:
+    """Create an Image2DArtifactModel instance."""
+    return Image2DArtifactModel(
+        name=name,
+        data=data,
+        format=format.lower(),
+        description=description,
+        parent_id=parent_id,
+        width=width,
+        height=height,
+        metadata=metadata or {},
+    )
+
+def create_image_artifact_from_file(
+    path: str,
+    name: Optional[str] = None,
+    description: Optional[str] = None,
+    parent_id: Optional[ObjectId] = None,
+) -> Image2DArtifactModel:
+    """Create an Image2DArtifactModel instance from a file path."""
+    if not os.path.exists(path):
+        raise FileNotFoundError(f"Image file not found: {path}")
+
+    filename = os.path.basename(path)
+    if name is None:
+        name = filename
+
+    # Extract format from extension
+    _, ext = os.path.splitext(filename)
+    format = ext.lstrip(".").lower()
+    if not format:
+         format = "png" # Default
+
+    with open(path, "rb") as f:
+        data = f.read()
+
+    return create_image_artifact(
+        name=name,
+        data=data,
+        format=format,
+        description=description,
+        parent_id=parent_id
+    )

simstack/models/log_entry_model.py

@@ -0,0 +1,37 @@
+from datetime import datetime
+from enum import Enum
+from typing import Optional
+
+from odmantic import Model, Field
+
+from simstack.models import Parameters
+
+
+class LogLevel(str, Enum):
+    DEBUG = "DEBUG"
+    INFO = "INFO"
+    WARNING = "WARNING"
+    ERROR = "ERROR"
+    CRITICAL = "CRITICAL"
+
+
+class LogEntry(Model):
+    timestamp: datetime = Field(default_factory=datetime.utcnow)
+    level: LogLevel  # Use enum for validation
+    logger_name: str
+    message: str
+    module: str
+    function: str
+    line: int
+    task_id: Optional[str] = None
+    resource: Optional[str] = None
+    thread_name: Optional[str] = None
+    process_name: Optional[str] = None
+    exception_type: Optional[str] = None
+    exception_message: Optional[str] = None
+    exception_traceback: Optional[str] = None
+    # extra_data: Dict[str, Any] = Field(default_factory=dict)
+    parameters: Optional[Parameters] = None  # If Parameters is related to LogEntry
+
+    # Replace Config class with model_config
+    model_config = {"collection": "logs"}

simstack/models/models.py

@@ -0,0 +1,52 @@
+import base64
+import logging
+from typing import List, Optional, TypeVar
+
+from odmantic import EmbeddedModel, Field, Model
+
+from simstack.models.parameters import Parameters
+from simstack.models.pickle_models import FunctionPickle
+
+logger = logging.getLogger("Models")
+
+T = TypeVar("T")
+
+
+class ModelMapping(Model):
+    """
+    name: shorthand - must be unique
+    mapping: full name - path relative to project root in module.module.class/function format
+    """
+
+    name: str = Field(unique=True)
+    mapping: str = Field(unique=True)
+    version: Optional[str] = None
+    collection_name: str
+    json_schema: Optional[str] = None
+    ui_schema: Optional[str] = None
+
+
+class DataMapping(EmbeddedModel):
+    name: str
+    mapping: str
+    description: Optional[str] = ""
+    version: Optional[str] = None
+
+
+class NodeModel(Model):
+    name: str = Field(unique=True)
+    function_mapping: str = Field(unique=True)
+    version: Optional[str] = None
+    input_mappings: List[DataMapping]
+    result_mappings: List[DataMapping] = Field(default_factory=list)
+    called_nodes: List[str] = Field(default_factory=list)
+    description: Optional[str] = ""
+    favorite: bool = False
+    default_parameters: Parameters
+    # Reference to FunctionPickle if available.
+    pickle_function: Optional[FunctionPickle] = None
+
+    model_config = {
+        "collection": "node_model",
+        "json_encoders": {bytes: lambda b: base64.b64encode(b).decode("ascii")},
+    }

simstack/models/named_data_reference.py

@@ -0,0 +1,36 @@
+from typing import Optional
+
+from odmantic import EmbeddedModel, ObjectId, Model
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+class NamedDataReference(EmbeddedModel):
+    variable_name: str
+    variable_mapping: str
+    reference: ObjectId
+    
+    def __repr__(self) -> str:
+        return f"NamedDataReference(variable_name='{self.variable_name}', variable_mapping='{self.variable_mapping}', reference='{self.reference}')"
+    
+    @classmethod
+    def from_variable(cls, variable: Model, variable_name: Optional[str] = None,  task_id: Optional[str] = None):
+        from simstack.core.context import context
+        if not isinstance(variable, Model):
+            logger.error(f"task_id: {task_id} Processing Variable {variable} must be an instance of Model")
+            raise ValueError(f"Variable must be an instance of Model")
+
+        variable_class_name = variable.__class__.__name__ 
+        table_name = context.model_mappings.get_by_name(variable_class_name)
+        if table_name is None:
+            logger.error(f"Could not find table name for {variable_class_name}")
+            raise ValueError(f"Could not find table name for {variable_class_name}")
+        if variable_name is None:
+            # Check if variable already has a field_name we can use
+            if hasattr(variable, "field_name") and variable.field_name:
+                variable_name = variable.field_name
+            else:
+                variable_name = "variable"
+
+        return cls(variable_name=variable_name, variable_mapping=table_name.mapping, reference=variable.id)

simstack/models/node_registry.py

@@ -0,0 +1,96 @@
+from datetime import datetime
+from typing import Optional, List
+
+from odmantic import Model, Field, ObjectId, Reference
+
+from simstack.core.definitions import TaskStatus
+from simstack.models.file_list import FileList
+from simstack.models.parameters import Parameters
+from simstack.models.named_data_reference import NamedDataReference
+
+class NodeRegistry(Model):
+    """
+    Represents a registry for nodes with associated metadata, configurations, and status
+    information. It allows tracking the state and attributes of a workflow node, including its
+    execution parameters, process status, and relationship to other nodes.
+
+    This class is designed for managing workflow instances and their lifecycle, with
+    capabilities to monitor execution states, inputs, outputs, and associated timestamps.
+
+    :ivar name: The name of the node.
+    :type name: str
+    :ivar custom_name: An optional custom name for the node.
+    :type custom_name: Optional[str]
+    :ivar status: TheTaskStatus of the node in string format.
+    :type status: TaskStatus
+    :ivar category: An optional category classification for the node.
+    :type category: Optional[str]
+    :ivar description: An optional description providing details about the node.
+    :type description: Optional[str]
+    :ivar input_references: List of references to input data.
+    :type input_references: List[NamedDataReference]
+    :ivar results_references: List of references to result data.
+    :type results_references: List[NamedDataReference]
+    :ivar parent_ids: A list of identifiers representing parent nodes associated with this node.
+    :type parent_ids: List[ObjectId]
+    :ivar created_at: The timestamp when the node was created.
+    :type created_at: datetime
+    :ivar started_at: An optional timestamp indicating when the execution of the node started.
+    :type started_at: Optional[datetime]
+    :ivar completed_at: An optional timestamp indicating when the execution of the node was
+                        completed.
+    :type completed_at: Optional[datetime]
+    :ivar function_hash: A hash value representing the unique function executed by this node.
+    :type function_hash: str
+    :ivar arg_hash: A hash value representing the unique arguments passed to the function of
+                    this node.
+    :type arg_hash: str
+    :ivar func_mapping: A mapping identifier associated with the function executed by this
+                        node.
+    :type func_mapping: str
+    :ivar is_async: A boolean value indicating whether the node execution is asynchronous.
+    :type is_async: bool
+    :ivar parameters: Parameters associated with the node execution.
+    :type parameters: Parameters
+    :ivar call_path: An optional path indicating where the function is called from by concatenating the name of all nodes in the call stack.
+    :type call_path: Optional[str]
+    """
+
+    name: str
+    status: TaskStatus
+    custom_name: Optional[str] = None
+    version: Optional[str] = None
+    # Keep this as Optional[ObjectId] instead of Reference(Project):
+    # in the ODMantic version used here, nullable references are not supported
+    # as Optional[Project] + Reference() field definitions.
+    project: Optional[ObjectId] = Field(default=None)
+    category: Optional[str] = None
+    description: Optional[str] = None
+    call_path: Optional[str] = None
+    assignment_rule_id: Optional[str] = None
+    assignment_rule_name: Optional[str] = None
+    assignment_pattern: Optional[str] = None
+    error: Optional[str] = None
+    message: Optional[str] = None
+
+    input_references: List[NamedDataReference] = Field(default_factory=list)
+    results_references: List[NamedDataReference] = Field(default_factory=list)
+
+    info_files: FileList = Field(default_factory=FileList)
+
+    parent_ids: List[ObjectId] = Field(default_factory=list)
+    artifact_ids: List[ObjectId] = Field(default_factory=list)
+    created_at: datetime = Field(default_factory=datetime.now)
+    started_at: Optional[datetime] = Field(default=None)
+    completed_at: Optional[datetime] = Field(default=None)
+    job_id: Optional[str] = None
+    function_hash: str
+    arg_hash: str
+    func_mapping: str
+    is_async: bool = False
+    parameters: Parameters = Reference()
+
+
+async def find_child_nodes(task_id: str) -> List[NodeRegistry]:
+    from simstack.core.context import context
+    return await context.db.find(NodeRegistry, {"parent_ids": {"$in": [task_id]}})

simstack/models/pandas_model.py

@@ -0,0 +1,239 @@
+import asyncio
+import json
+import io
+from pprint import pprint
+from typing import Any, Dict, Optional
+
+import numpy as np
+import pandas as pd
+from odmantic import Model
+
+from simstack.core.context import context
+from simstack.models import simstack_model
+from simstack.models.files import FileStack
+
+
+def _format_df_for_console(df: pd.DataFrame, precision: int = 2, max_rows: int = 60, max_cols: int = 30) -> str:
+    """
+    Console-safe DataFrame formatting (no optional deps like jinja2 required).
+    """
+    if df is None or df.empty:
+        return "Empty DataFrame"
+
+    df_out = df.copy()
+
+    # Round only numeric columns to avoid touching strings/datetimes
+    numeric_cols = df_out.select_dtypes(include=["number"]).columns
+    if len(numeric_cols) > 0:
+        df_out[numeric_cols] = df_out[numeric_cols].round(precision)
+
+    with pd.option_context(
+        "display.max_rows", max_rows,
+        "display.max_columns", max_cols,
+        "display.width", 0,  # auto-detect width
+        "display.max_colwidth", 200,
+    ):
+        return df_out.to_string(index=False)
+
+
+def _format_datetime_columns(df):
+    """
+    Format all datetime columns to match the expected format: YYYY-MM-DDThh:mm:ss
+    Returns a copy of the dataframe with formatted datetime columns.
+    """
+    df_copy = df.copy()
+    # Format datetime columns to match expected format without microseconds and Z
+    for col in df_copy.select_dtypes(include=["datetime64"]):
+        df_copy[col] = df_copy[col].dt.strftime("%Y-%m-%dT%H:%M:%S")
+    return df_copy
+
+
+@simstack_model
+class PandasModel(Model):
+    model_config = {"indexes": [("field_name", {"unique": True})]}
+
+    field_name: str = "pandas_model"
+    content_: bytes = b""
+    file_stack: Optional[FileStack] = None
+
+    @classmethod
+    def from_data_frame(cls, df):
+        new_instance = cls()
+        new_instance.table = df
+        return new_instance
+
+    @property
+    def table(self):
+        if not self.content_:
+            return pd.DataFrame()
+
+        # Create a BytesIO object from the binary content
+        buffer = io.BytesIO(self.content_)
+
+        # Use pandas read_pickle to decompress and load the DataFrame
+        return pd.read_pickle(buffer)
+
+    @table.setter
+    def table(self, df):
+        if not isinstance(df, pd.DataFrame):
+            raise TypeError("Expected a pandas DataFrame")
+
+        # Create a BytesIO object to store the binary content
+        buffer = io.BytesIO()
+
+        # Serialize the DataFrame to the buffer
+        df.to_pickle(buffer)
+
+        # Get the binary content from the buffer
+        self.content_ = buffer.getvalue()
+
+    def to_react_json(self, orient="records"):
+        """
+        Convert the DataFrame to a JSON string suitable for React visualization libraries.
+
+        Parameters:
+        - orient: Determines the JSON string layout:
+          'records' - list like [{column -> value}, ... , {column -> value}] (default)
+          'columns' - {column -> [values, ...]}
+          'index'   - {column -> value}}
+          'split'   - {index -> [index], columns -> [columns], data -> [values]}
+          'table'   - {'schema': {schema}, 'data': {data}}
+
+        Returns:
+        - String: JSON formatted string ready for React
+        """
+        df = self.table
+        if df.empty:
+            return json.dumps([])
+
+        # Format datetime columns
+        df = _format_datetime_columns(df)
+
+        # Create a custom serializer to handle NumPy types properly
+        class NumpyEncoder(json.JSONEncoder):
+            def default(self, obj):
+                if isinstance(obj, np.integer):
+                    return int(obj)  # Keep integers as integers
+                elif isinstance(obj, np.floating):
+                    return float(obj)  # Convert numpy float to Python float
+                elif isinstance(obj, np.ndarray):
+                    return obj.tolist()  # Convert arrays to lists
+                elif pd.isna(obj):
+                    return None  # Convert NaN/NaT to None
+                return super().default(obj)
+
+        # Convert DataFrame to dictionary while preserving data types
+        if orient == "records":
+            data = df.to_dict(orient="records")
+        else:
+            data = df.to_dict(orient=orient)
+
+        # Use custom JSON encoder to handle NumPy types properly
+        return json.dumps(data, cls=NumpyEncoder)
+
+    def to_react_data(self, orient="records"):
+        """
+        Convert the DataFrame to a Python object suitable for conversion to JSON.
+        This can be used in API responses.
+
+        Returns:
+        - List/Dict: Python object ready for json.dumps()
+        """
+        df = self.table
+        if df.empty:
+            return []
+
+        # Format datetime columns using the same helper method
+        df = _format_datetime_columns(df)
+
+        # Convert DataFrame to dictionary while preserving data types
+        if orient == "records":
+            data = df.to_dict(orient="records")
+        else:
+            data = df.to_dict(orient=orient)
+
+        # Replace NaN values with None
+        if isinstance(data, list):
+            for item in data:
+                for key, value in item.items():
+                    if pd.isna(value):
+                        item[key] = None
+        elif isinstance(data, dict):
+            for key, values in data.items():
+                if isinstance(values, dict):
+                    for sub_key, value in values.items():
+                        if pd.isna(value):
+                            values[sub_key] = None
+                elif isinstance(values, list):
+                    data[key] = [None if pd.isna(v) else v for v in values]
+
+        return data
+
+    async def custom_model_dump(self, **kwargs) -> Dict[str, Any]:
+        dumped_data = self.to_react_data("dict")
+        # del dumped_data["content"]  # Exclude content from the dumped data
+        return dumped_data
+
+    def __repr__(self):
+        if not self.content_:
+            return "PandasModel(empty table)"
+
+        df = self.table
+        rows, cols = df.shape
+        return f"PandasModel({rows} rows × {cols} columns)"
+
+    def __str__(self):
+        if not self.content_:
+            return "Empty pandas table"
+
+        df = self.table
+        if len(df) > 5:
+            return f"PandasModel with shape {df.shape}:\n{df.head(5).to_string()}\n..."
+        return f"PandasModel with shape {df.shape}:\n{df.to_string()}"
+
+#TODO move to test
+async def main():
+    await context.initialize()
+    # Create the data structure
+    data = []
+
+    # Iteration names
+    iterations = ["iter1", "iter2", "iter3"]
+
+    # For each iteration
+    for index, iteration in enumerate(iterations):
+        # Generate 4 x values (for example, increasing by 0.5)
+        x_values = np.arange(1, 3, 0.5)  # Creates [1.0, 1.5, 2.0, 2.5]
+
+        # Generate 2 sets of 4 y values for each iteration
+        y_values_set1 = np.sin((index + 1) * x_values)  # 4 random values around mean=8
+        y_values_set2 = np.sin((index + 1) * x_values)
+        # Add the data for this iteration
+        for i, x in enumerate(x_values):
+            data.append(
+                {
+                    "iteration": iteration,
+                    "x": x,
+                    "y_set1": y_values_set1[i],
+                    "y_set2": y_values_set2[i],
+                }
+            )
+
+    # Create the DataFrame
+    df = pd.DataFrame(data)
+    # Replace Styler usage (requires jinja2) with console-safe formatting.
+    print(_format_df_for_console(df, precision=2))
+
+    model = PandasModel.from_data_frame(df)
+
+    pprint(await model.custom_model_dump())
+    saved_model = await context.db.save(model)
+
+    retrieved_model = await context.db.find_one(
+        PandasModel, PandasModel.id == saved_model.id
+    )
+    print("Retrieved Model", retrieved_model)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())