simstack 0.2.0__py3-none-any.whl

simstack/models/files.py

@@ -0,0 +1,229 @@
+import logging
+import os
+import tempfile
+import zlib
+from pathlib import Path
+from typing import List, Optional, Union, Dict, Any
+
+from odmantic import Model, Field, ObjectId
+
+
+from simstack.models import simstack_model
+from simstack.models.file_instance import FileInstance
+from simstack.models.parameters import Resource
+from simstack.util.file_hashing import hash_file
+
+logger = logging.getLogger(__name__)
+
+
+@simstack_model
+class FileStack(Model):
+    name: Optional[str] = Field(description="Name of the file", default=None)
+    size: Optional[int] = Field(description="Size of the file in bytes", default=None)
+    is_hashable: bool = Field(
+        default=False, description="Whether the file stack is hashable"
+    )
+    hash: Optional[str] = Field(
+        default=None, description="Hash of the file stack content", index=True
+    )
+    in_memory: bool = Field(
+        default=False, description="Whether the file stack is in memory"
+    )
+    content: Optional[bytes] = Field(
+        description="Compressed file content", default=None
+    )
+
+    locations: List[FileInstance] = Field(
+        default_factory=list, description="List of file locations"
+    )
+
+    def __str__(self):
+        return f"FileStack(name={self.name}, size={self.size}, is_hashable={self.is_hashable}, in_memory={self.in_memory}, locations={self.locations})"
+
+    def __repr__(self):
+        return f"FileStack(name={self.name}, size={self.size}, is_hashable={self.is_hashable}, in_memory={self.in_memory}, locations={self.locations})"
+
+    async def custom_model_dump(self, **kwargs) -> Dict[str, Any]:
+        dumped_data = self.model_dump()
+        del dumped_data["content"]  # Exclude content from the dumped data
+        return dumped_data
+
+    @classmethod
+    def ui_base_schema(cls, **kwargs) -> Dict[str, Any]:
+        # TODO get the model programatically
+        return {
+            "ui:field": "FileField",
+            "ui:options": {
+                "model": "simstack.models.files.FileStack",
+            },
+        }
+
+    @classmethod
+    def from_local_file(
+        cls,
+        path: Union[Path, str],
+        is_hashable: bool = True,
+        in_memory: bool = True,
+        secure_source: bool = False,
+        task_id: str = "",
+    ):
+        """
+        Creates a FileStack object from a local file path.
+
+        :param task_id: task_id of the task that created the file stack, used for logging and tracking
+        :type task_id: str
+        :param secure_source: specifies if the source is secure (already in a directory generated within Simstack II)
+        :type secure_source: bool
+        :param path: The path to the local file. Can be provided as a string or Path object.
+        :type path: Union[Path, str]
+        :param is_hashable: A flag indicating whether the file hash needs to be calculated.
+        :type is_hashable: bool
+        :param in_memory: Whether to store the compressed file content in memory. Defaults to True.
+        :type in_memory: bool
+
+        :return: A FileStack object containing FileInstances for the file.
+        :rtype: FileStack
+        """
+        # TODO make a second version where a unique directory is already there
+        source_path = path if isinstance(path, Path) else Path(path)
+
+        # Check if the source exists
+        if not source_path.exists():
+            logger.error(f"Source file not found: {source_path}")
+            raise FileNotFoundError(f"Source file not found: {source_path}")
+
+        # Check if it's a file (not a directory)
+        if not source_path.is_file():
+            logger.error(f"Source file {path} is not a file")
+            raise ValueError(f"Source file is not a file: {path}")
+
+        # Check read permission using os.access
+        if not os.access(path, os.R_OK):
+            logger.error(f"No read permission for file: {path}")
+            raise ValueError(f"No read permission for file: {path}")
+
+        content = None
+        file_hash = hash_file(source_path) if is_hashable else None
+        name = source_path.name
+        size = source_path.stat().st_size if source_path.exists() else None
+
+        if in_memory:
+            try:
+                # Read the file content
+                with open(source_path, "rb") as f:
+                    file_content = f.read()
+                # Compress the content using zlib
+                content = zlib.compress(file_content)
+                logger.debug(
+                    f"Compressed file {source_path} from {len(file_content)} bytes to {len(content)} bytes"
+                )
+            except Exception as e:
+                logger.warning(f"Failed to compress file {source_path}: {e}")
+
+        file_stack = cls(
+            name=name,
+            in_memory=in_memory,
+            content=content,
+            is_hashable=is_hashable,
+            size=size,
+            hash=file_hash,
+        )
+
+        if not in_memory:
+            location = FileInstance.from_local_file(
+                path=path, file_stack_id=file_stack.id, make_copy=not secure_source
+            )
+            file_stack.locations.append(location)
+        return file_stack
+
+    def complex_hash(self) -> str:
+        if self.is_hashable:
+            if self.hash:
+                return self.hash
+            else:
+                raise ValueError("FileStack is hashable but hash is not set.")
+            #elif self.in_memory and self.content:
+            #    # If the content is in memory, hash the compressed content
+            #    return complex_hash_function(zlib.decompress(self.content))
+            #else:
+            #    temp_dir = Path(tempfile.mkdtemp())
+            #    local_file = self.get(None, local_dir=temp_dir)
+            #    return complex_hash_function(local_file.read_bytes())
+        else:
+            logger.warning(
+                f"FileStack {self.id} is not hashable, returning unique hash."
+            )
+            return str(ObjectId())
+
+    def append(self, file_instance: FileInstance):
+        """
+        Appends a FileInstance to the file stack.
+
+        :param file_instance: The FileInstance to append.
+        :type file_instance: FileInstance
+        """
+        self.locations.append(file_instance)
+
+    def get(self, local_resource: Resource, local_dir: Path) -> Path:
+        """
+        Copies the file stack to a local directory.
+
+        :param local_resource: the local resource to copy the file stack to. Defaults to the current resource.
+        :param local_dir: The local directory to copy the file stack to.
+        :type local_dir: Path
+        """
+
+        # select the best instance
+        # first search for an instance with "in_memory" set to True
+
+        if self.in_memory:
+            local_dir.mkdir(parents=True, exist_ok=True)
+            try:
+                # Decompress the content
+                decompressed_content = zlib.decompress(self.content)
+                # Write the decompressed content to the local directory
+                with open(local_dir / self.name, "wb") as f:
+                    f.write(decompressed_content)
+                return local_dir / self.name
+            except Exception as e:
+                logger.error(f"Failed to decompress and write file {self.name}: {e}")
+                raise ValueError(
+                    f"Failed to decompress and write file {self.name}: {e}"
+                )
+        # If in-memory instance not found or decompression failed, try finding instance on same resource
+        if same_resource_instance := next(
+            (f for f in self.locations if f.resource == local_resource), None
+        ):
+            # Copy the file from the source path to the destination path
+            return Path(same_resource_instance.path)
+        else:
+            local_dir.mkdir(parents=True, exist_ok=True)
+            logger.error("No suitable file instance found for copying.")
+            from simstack.methods.get_file import get_file
+            return get_file(self,local_resource, local_dir / self.name)
+
+
+    def str(self):
+        return f"FileStack(name={self.name}, size={self.size}, is_hashable={self.is_hashable}, in_memory={self.in_memory}, locations={self.locations})"
+
+
+async def main():
+    from simstack.core.context import context
+
+    context.initialize()
+    # write a file test.txt
+    with open("test.txt", "w") as f:
+        f.write("Hello World")
+    file_stack = FileStack.from_local_file("test.txt", is_hashable=True, in_memory=True)
+    print(file_stack)
+    await context.db.save(file_stack)
+    local_dir = Path(context.config.workdir) / "samira" / str(file_stack.id)
+    retrieved = file_stack.get(context.config.resource, local_dir=local_dir)
+    print("Retrieved file path:", retrieved)
+
+
+if __name__ == "__main__":
+    import asyncio
+
+    logging.basicConfig(level=logging.DEBUG)
+    asyncio.run(main())

simstack/models/log_entry_model.py

@@ -0,0 +1,37 @@
+from datetime import datetime
+from enum import Enum
+from typing import Optional
+
+from odmantic import Model, Field
+
+from simstack.models import Parameters
+
+
+class LogLevel(str, Enum):
+    DEBUG = "DEBUG"
+    INFO = "INFO"
+    WARNING = "WARNING"
+    ERROR = "ERROR"
+    CRITICAL = "CRITICAL"
+
+
+class LogEntry(Model):
+    timestamp: datetime = Field(default_factory=datetime.utcnow)
+    level: LogLevel  # Use enum for validation
+    logger_name: str
+    message: str
+    module: str
+    function: str
+    line: int
+    task_id: Optional[str] = None
+    resource: Optional[str] = None
+    thread_name: Optional[str] = None
+    process_name: Optional[str] = None
+    exception_type: Optional[str] = None
+    exception_message: Optional[str] = None
+    exception_traceback: Optional[str] = None
+    # extra_data: Dict[str, Any] = Field(default_factory=dict)
+    parameters: Optional[Parameters] = None  # If Parameters is related to LogEntry
+
+    # Replace Config class with model_config
+    model_config = {"collection": "logs"}

simstack/models/models.py

@@ -0,0 +1,40 @@
+import base64
+from typing import Optional, List, TypeVar
+from odmantic import Model, Field
+from simstack.models.parameters import Parameters
+from simstack.models.pickle_models import ClassPickle, FunctionPickle
+import logging
+
+logger = logging.getLogger("Models")
+
+T = TypeVar("T")
+
+
+class ModelMapping(Model):
+    """
+    name: shorthand - must be unique
+    mapping: full name - path relative to project root in module.module.class/function format
+    """
+    name: str = Field(unique=True)
+    mapping: str = Field(unique=True)
+    collection_name: str
+    json_schema: Optional[str] = None
+    ui_schema: Optional[str] = None
+
+
+
+class NodeModel(Model):
+    name: str = Field(unique=True)
+    function_mapping: str = Field(unique=True)
+    input_mappings: List[str]
+    description: Optional[str] = ""
+    favorite: bool = False
+    default_parameters: Parameters
+    pickle_function: Optional[
+        FunctionPickle
+    ] = None  # Reference to FunctionPickle if available
+
+    model_config = {
+        "collection": "node_model",
+        "json_encoders": {bytes: lambda b: base64.b64encode(b).decode("ascii")},
+    }

simstack/models/node_registry.py

@@ -0,0 +1,105 @@
+from datetime import datetime
+from typing import Optional, List
+
+from odmantic import Model, Field, ObjectId, Reference
+
+from simstack.core.definitions import TaskStatus
+from simstack.models.file_list import FileList
+from simstack.models.parameters import Parameters
+from simstack.core.engine import current_engine_context
+
+
+class NodeRegistry(Model):
+    """
+    Represents a registry for nodes with associated metadata, configurations, and status
+    information. It allows tracking the state and attributes of a workflow node, including its
+    execution parameters, process status, and relationship to other nodes.
+
+    This class is designed for managing workflow instances and their lifecycle, with
+    capabilities to monitor execution states, inputs, outputs, and associated timestamps.
+
+    :ivar name: The name of the node.
+    :type name: str
+    :ivar custom_name: An optional custom name for the node.
+    :type custom_name: Optional[str]
+    :ivar status: TheTaskStatus of the node in string format.
+    :type status: TaskStatus
+    :ivar category: An optional category classification for the node.
+    :type category: Optional[str]
+    :ivar description: An optional description providing details about the node.
+    :type description: Optional[str]
+    :ivar input_tables: The names of the classes associated with this node's inputs.
+    :type input_tables: List[str]
+    :ivar input_ids: The unique identifier for the specific workflow instance providing the
+                     inputs.
+    :type input_ids: List[ObjectId]
+    :ivar result_tables: list of names of the result class of the ode's outputs.
+    :type result_tables: List[str]
+    :ivar result_ids: List of ids for the results instance.
+    :type result_ids: ListObjectId]
+    :ivar parent_ids: A list of identifiers representing parent nodes associated with this node.
+    :type parent_ids: List[ObjectId]
+    :ivar created_at: The timestamp when the node was created.
+    :type created_at: datetime
+    :ivar started_at: An optional timestamp indicating when the execution of the node started.
+    :type started_at: Optional[datetime]
+    :ivar completed_at: An optional timestamp indicating when the execution of the node was
+                        completed.
+    :type completed_at: Optional[datetime]
+    :ivar function_hash: A hash value representing the unique function executed by this node.
+    :type function_hash: str
+    :ivar arg_hash: A hash value representing the unique arguments passed to the function of
+                    this node.
+    :type arg_hash: str
+    :ivar func_mapping: A mapping identifier associated with the function executed by this
+                        node.
+    :type func_mapping: str
+    :ivar is_async: A boolean value indicating whether the node execution is asynchronous.
+    :type is_async: bool
+    :ivar parameters: Parameters associated with the node execution.
+    :type parameters: Parameters
+    :ivar call_path: An optional path indicating where the function is called from by concatenating the name of all nodes in the call stack.
+    :type call_path: Optional[str]
+    """
+
+    name: str
+    status: TaskStatus
+    custom_name: Optional[str] = None
+    project: Optional[str] = "default"
+    category: Optional[str] = None
+    description: Optional[str] = None
+    call_path: Optional[str] = None
+    error: Optional[str] = None
+    message: Optional[str] = None
+
+    input_names: List[str] = Field(default_factory=list)
+    input_tables: List[str] = Field(
+        default_factory=list
+    )  # The model mappings the input classes
+    input_ids: List[ObjectId] = Field(
+        default_factory=list
+    )  # The ID of the specific workflow instance
+    result_tables: List[str] = Field(
+        default_factory=list
+    )  # The model mappings of the result classes
+    result_ids: List[ObjectId] = Field(default_factory=list)
+    result_names: List[str] = Field(default_factory=list)
+
+    info_files: FileList = Field(default=FileList())
+
+    parent_ids: List[ObjectId] = Field(default_factory=list)
+    artifact_ids: List[ObjectId] = Field(default_factory=list)
+    created_at: datetime = Field(default_factory=datetime.now)
+    started_at: Optional[datetime] = Field(default=None)
+    completed_at: Optional[datetime] = Field(default=None)
+    job_id: Optional[str] = None
+    function_hash: str
+    arg_hash: str
+    func_mapping: str
+    is_async: bool = False
+    parameters: Parameters = Reference()
+
+
+async def find_child_nodes(task_id: str) -> List[NodeRegistry]:
+    engine = current_engine_context.get()
+    return await engine.find(NodeRegistry, {"parent_ids": {"$in": [task_id]}})

simstack/models/pandas_model.py

@@ -0,0 +1,210 @@
+import asyncio
+import io
+import json
+from pprint import pprint
+from typing import Dict, Any
+
+import numpy as np
+import pandas as pd
+from odmantic import Model
+
+from simstack.core.context import context
+from simstack.models import simstack_model
+
+
+def _format_datetime_columns(df):
+    """
+    Format all datetime columns to match the expected format: YYYY-MM-DDThh:mm:ss
+    Returns a copy of the dataframe with formatted datetime columns.
+    """
+    df_copy = df.copy()
+    # Format datetime columns to match expected format without microseconds and Z
+    for col in df_copy.select_dtypes(include=["datetime64"]):
+        df_copy[col] = df_copy[col].dt.strftime("%Y-%m-%dT%H:%M:%S")
+    return df_copy
+
+
+@simstack_model
+class PandasModel(Model):
+    content_: bytes = b""
+
+    @classmethod
+    def from_data_frame(cls, df):
+        new_instance = cls()
+        new_instance.table = df
+        return new_instance
+
+    @property
+    def table(self):
+        if not self.content_:
+            return pd.DataFrame()
+
+        # Create a BytesIO object from the binary content
+        buffer = io.BytesIO(self.content_)
+
+        # Use pandas read_pickle to decompress and load the DataFrame
+        return pd.read_pickle(buffer)
+
+    @table.setter
+    def table(self, df):
+        if not isinstance(df, pd.DataFrame):
+            raise TypeError("Expected a pandas DataFrame")
+
+        # Create a BytesIO object to store the binary content
+        buffer = io.BytesIO()
+
+        # Serialize the DataFrame to the buffer
+        df.to_pickle(buffer)
+
+        # Get the binary content from the buffer
+        self.content_ = buffer.getvalue()
+
+    def to_react_json(self, orient="records"):
+        """
+        Convert the DataFrame to a JSON string suitable for React visualization libraries.
+
+        Parameters:
+        - orient: Determines the JSON string layout:
+          'records' - list like [{column -> value}, ... , {column -> value}] (default)
+          'columns' - {column -> [values, ...]}
+          'index'   - {column -> value}}
+          'split'   - {index -> [index], columns -> [columns], data -> [values]}
+          'table'   - {'schema': {schema}, 'data': {data}}
+
+        Returns:
+        - String: JSON formatted string ready for React
+        """
+        df = self.table
+        if df.empty:
+            return json.dumps([])
+
+        # Format datetime columns
+        df = _format_datetime_columns(df)
+
+        # Create a custom serializer to handle NumPy types properly
+        class NumpyEncoder(json.JSONEncoder):
+            def default(self, obj):
+                if isinstance(obj, np.integer):
+                    return int(obj)  # Keep integers as integers
+                elif isinstance(obj, np.floating):
+                    return float(obj)  # Convert numpy float to Python float
+                elif isinstance(obj, np.ndarray):
+                    return obj.tolist()  # Convert arrays to lists
+                elif pd.isna(obj):
+                    return None  # Convert NaN/NaT to None
+                return super().default(obj)
+
+        # Convert DataFrame to dictionary while preserving data types
+        if orient == "records":
+            data = df.to_dict(orient="records")
+        else:
+            data = df.to_dict(orient=orient)
+
+        # Use custom JSON encoder to handle NumPy types properly
+        return json.dumps(data, cls=NumpyEncoder)
+
+    def to_react_data(self, orient="records"):
+        """
+        Convert the DataFrame to a Python object suitable for conversion to JSON.
+        This can be used in API responses.
+
+        Returns:
+        - List/Dict: Python object ready for json.dumps()
+        """
+        df = self.table
+        if df.empty:
+            return []
+
+        # Format datetime columns using the same helper method
+        df = _format_datetime_columns(df)
+
+        # Convert DataFrame to dictionary while preserving data types
+        if orient == "records":
+            data = df.to_dict(orient="records")
+        else:
+            data = df.to_dict(orient=orient)
+
+        # Replace NaN values with None
+        if isinstance(data, list):
+            for item in data:
+                for key, value in item.items():
+                    if pd.isna(value):
+                        item[key] = None
+        elif isinstance(data, dict):
+            for key, values in data.items():
+                if isinstance(values, dict):
+                    for sub_key, value in values.items():
+                        if pd.isna(value):
+                            values[sub_key] = None
+                elif isinstance(values, list):
+                    data[key] = [None if pd.isna(v) else v for v in values]
+
+        return data
+
+    async def custom_model_dump(self, **kwargs) -> Dict[str, Any]:
+        dumped_data = self.to_react_data("dict")
+        # del dumped_data["content"]  # Exclude content from the dumped data
+        return dumped_data
+
+    def __repr__(self):
+        if not self.content_:
+            return "PandasModel(empty table)"
+
+        df = self.table
+        rows, cols = df.shape
+        return f"PandasModel({rows} rows × {cols} columns)"
+
+    def __str__(self):
+        if not self.content_:
+            return "Empty pandas table"
+
+        df = self.table
+        if len(df) > 5:
+            return f"PandasModel with shape {df.shape}:\n{df.head(5).to_string()}\n..."
+        return f"PandasModel with shape {df.shape}:\n{df.to_string()}"
+
+
+async def main():
+    context.initialize()
+    # Create the data structure
+    data = []
+
+    # Iteration names
+    iterations = ["iter1", "iter2", "iter3"]
+
+    # For each iteration
+    for index, iteration in enumerate(iterations):
+        # Generate 4 x values (for example, increasing by 0.5)
+        x_values = np.arange(1, 3, 0.5)  # Creates [1.0, 1.5, 2.0, 2.5]
+
+        # Generate 2 sets of 4 y values for each iteration
+        y_values_set1 = np.sin((index + 1) * x_values)  # 4 random values around mean=8
+        y_values_set2 = np.sin((index + 1) * x_values)
+        # Add the data for this iteration
+        for i, x in enumerate(x_values):
+            data.append(
+                {
+                    "iteration": iteration,
+                    "x": x,
+                    "y_set1": y_values_set1[i],
+                    "y_set2": y_values_set2[i],
+                }
+            )
+
+    # Create the DataFrame
+    df = pd.DataFrame(data)
+    print(df.style.format(precision=2))
+
+    model = PandasModel.from_data_frame(df)
+
+    pprint(await model.custom_model_dump())
+    saved_model = await context.db.save(model)
+
+    retrieved_model = await context.db.find_one(
+        PandasModel, PandasModel.id == saved_model.id
+    )
+    print("Retrieved Model", retrieved_model)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())