simstack 0.2.0__py3-none-any.whl

simstack/models/dataset_metadata.py

@@ -0,0 +1,303 @@
+from datetime import datetime
+from typing import Dict, Any, Union, List
+
+from odmantic import Model, EmbeddedModel, Field
+
+from simstack.core.asnyc_helper import async_helper
+from simstack.core.context import context
+from simstack.core.engine import current_engine_context
+from simstack.models import simstack_model
+
+
+def _get_json_schema(data: Dict) -> dict:
+    """
+    Inspects the current data dict and returns a JSON schema for that dict.
+
+    Returns:
+        dict: JSON schema describing the structure and types of the current data dict
+    """
+    if not data:
+        return {"type": "object", "properties": {}, "additionalProperties": False}
+
+    properties = {}
+
+    for key, value in data.items():
+        if isinstance(value, str):
+            properties[key] = {"type": "string"}
+        elif isinstance(value, bool):
+            # Check bool before int/float since bool is a subclass of int in Python
+            properties[key] = {"type": "boolean"}
+        elif isinstance(value, int):
+            properties[key] = {"type": "integer"}
+        elif isinstance(value, float):
+            properties[key] = {"type": "number"}
+        elif isinstance(value, datetime):
+            properties[key] = {"type": "string", "format": "date-time"}
+        else:
+            # Fallback for any unexpected types
+            properties[key] = {"type": "string"}
+
+    schema = {"type": "object", "properties": properties, "additionalProperties": False}
+
+    return schema
+
+
+class DataSetMetadataTemplate(Model):
+    dataset_type: str
+    model_json: dict[str, Any]
+    structure: Dict[str, List[str]] = Field(default_factory=dict)
+
+
+@simstack_model
+class DataSetMetadata(EmbeddedModel):
+    dataset_type: str = Field(unique=True)
+    data: Dict[str, Union[str, int, float, bool, datetime]] = Field(
+        default_factory=dict
+    )
+    is_validated: bool = False
+    structure: Dict[str, List[str]] = Field(default_factory=dict)
+
+    def get_json_schema(self):
+        return _get_json_schema(self.data)
+
+    async def validate_dict(self, new_structure: Dict[str, List[str]]) -> bool:
+        engine = current_engine_context.get()
+        reference_metadata = await engine.find_one(
+            DataSetMetadataTemplate,
+            DataSetMetadataTemplate.dataset_type == self.dataset_type,
+        )
+        if reference_metadata is None:
+            metadata_template = DataSetMetadataTemplate(
+                dataset_type=self.dataset_type,
+                model_json=_get_json_schema(self.data),
+                structure=new_structure,
+            )
+
+            await context.db.save(metadata_template)
+            return True  # first model of this type
+
+        new_data_json = _get_json_schema(self.data)
+        if reference_metadata.model_json != new_data_json:
+            raise ValueError(
+                f"Data schema has changed in the database reference: {reference_metadata.model_json} current: {new_data_json}"
+            )
+
+        # Check if lists in existing sections match
+        save_template = False
+        for section, content in new_structure.items():
+            if content:  # if there are no elements in a section this should be None
+                if section in reference_metadata.structure:
+                    if content != reference_metadata.structure[section]:
+                        raise ValueError(
+                            f"Section {section} has different content in existing structure"
+                        )
+                else:
+                    save_template = True
+                    self.structure[section] = content
+            else:
+                # if there is no content in a section, we can just copy it from the reference metadata
+                new_structure[section] = reference_metadata.structure[section]
+
+        if save_template:
+            reference_metadata.structure = new_structure
+            await engine.save(reference_metadata)
+        self.structure = new_structure
+        return True
+
+    @async_helper
+    async def freeze(self, new_structure: Dict[str, Dict[str, Any]]) -> bool:
+        engine = current_engine_context.get()
+        reference_metadata = await engine.find_one(
+            DataSetMetadataTemplate,
+            DataSetMetadataTemplate.dataset_type == self.dataset_type,
+        )
+        if not reference_metadata:
+            raise ValueError("Metadata does not exist")
+        if self.structure != reference_metadata.structure:
+            raise ValueError("Metadata structure has changed in the database")
+        if self.structure == {}:
+            self.structure = new_structure
+            reference_metadata.structure = new_structure
+            await engine.save(reference_metadata)
+        # some structure exists already
+        return new_structure == self.structure
+
+    @property
+    def initialized(self) -> bool:
+        """Check if the model has been fully constructed."""
+        # A simple heuristic: if we have an ID or if type is set, we're initialized
+        return hasattr(self, "dataset_type") and self.dataset_type is not None
+
+    # Dict-like behavior methods
+    def __getitem__(self, key: str):
+        """Get item from data dict."""
+        return self.data[key]
+
+    def __setitem__(self, key: str, value: Union[str, int, float, bool, datetime]):
+        """Set item in data dict with validation."""
+        # Validate value type
+        if not isinstance(value, (str, int, float, bool, datetime)):
+            raise TypeError(
+                f"Value must be str, int, float, bool, or datetime, got {type(value).__name__}"
+            )
+
+        # Only check for structural changes after initialization
+        if self.initialized and key not in self.data:
+            raise KeyError(
+                f"Cannot add new key '{key}' after initialization. Existing keys: {list(self.data.keys())}"
+            )
+
+        # Only check for type changes after initialization
+        if self.initialized and key in self.data:
+            existing_value = self.data[key]
+            if type(existing_value) != type(value):
+                raise TypeError(
+                    f"Cannot change type of key '{key}' from {type(existing_value).__name__} "
+                    f"to {type(value).__name__}"
+                )
+
+        self.data[key] = value
+
+    def __delitem__(self, key: str):
+        """Delete item from data dict."""
+        if self.initialized:
+            raise KeyError(f"Cannot delete key '{key}' after initialization")
+        del self.data[key]
+
+    def __contains__(self, key: str) -> bool:
+        """Check if key exists in data dict."""
+        return key in self.data
+
+    def __iter__(self):
+        """Iterate over keys in data dict."""
+        return iter(self.data)
+
+    def __len__(self) -> int:
+        """Get number of items in data dict."""
+        return len(self.data)
+
+    def keys(self):
+        """Get keys from data dict."""
+        return self.data.keys()
+
+    def values(self):
+        """Get values from data dict."""
+        return self.data.values()
+
+    def items(self):
+        """Get items from data dict."""
+        return self.data.items()
+
+    def get(self, key: str, default=None):
+        """Get item from data dict with default."""
+        return self.data.get(key, default)
+
+    def pop(self, key: str, *args):
+        """Pop item from data dict."""
+        if self.initialized:
+            raise KeyError(f"Cannot pop key '{key}' after initialization")
+        return self.data.pop(key, *args)
+
+    def popitem(self):
+        """Pop item from data dict."""
+        if self.initialized:
+            raise KeyError("Cannot pop items after initialization")
+        return self.data.popitem()
+
+    def clear(self):
+        """Clear data dict."""
+        if self.initialized:
+            raise RuntimeError("Cannot clear data after initialization")
+        self.data.clear()
+
+    def update(self, *args, **kwargs):
+        """Update data dict with validation."""
+        # Handle different update signatures
+        if args:
+            other = args[0]
+            if hasattr(other, "items"):
+                items_to_update = other.items()
+            else:
+                items_to_update = other
+        else:
+            items_to_update = []
+
+        # Combine with kwargs
+        all_items = list(items_to_update) + list(kwargs.items())
+
+        # Validate all items before updating
+        for key, value in all_items:
+            # Type validation (always required)
+            if not isinstance(value, (str, int, float, bool, datetime)):
+                raise TypeError(
+                    f"Value for key '{key}' must be str, int, float, bool, or datetime, got {type(value).__name__}"
+                )
+
+            # Structural validation (only after initialization)
+            if self.initialized:
+                if key not in self.data:
+                    raise KeyError(f"Cannot add new key '{key}' after initialization")
+
+                existing_value = self.data[key]
+                if type(existing_value) != type(value):
+                    raise TypeError(
+                        f"Cannot change type of key '{key}' from {type(existing_value).__name__} "
+                        f"to {type(value).__name__}"
+                    )
+
+        # If all validations pass, update the data
+        for key, value in all_items:
+            self.data[key] = value
+
+    def setdefault(self, key: str, default=None):
+        """Set default value for key if not exists."""
+        if key not in self.data:
+            if self.initialized:
+                raise KeyError(f"Cannot add new key '{key}' after initialization")
+
+            if default is not None and not isinstance(
+                default, (str, int, float, bool, datetime)
+            ):
+                raise TypeError(
+                    f"Default value must be str, int, float, bool, or datetime, got {type(default).__name__}"
+                )
+
+            self.data[key] = default
+
+        return self.data[key]
+
+    # Additional utility methods (no initialization checks needed)
+    def copy_data(self) -> Dict[str, Union[str, int, float, bool, datetime]]:
+        """Return a copy of the data dict."""
+        return self.data.copy()
+
+    def is_type_compatible(self, key: str, value) -> bool:
+        """Check if a value is type-compatible with existing key."""
+        if key not in self.data:
+            return isinstance(value, (str, int, float, bool, datetime))
+        return type(self.data[key]) == type(value)
+
+    def get_key_type(self, key: str) -> type:
+        """Get the type of a specific key."""
+        if key not in self.data:
+            raise KeyError(f"Key '{key}' not found")
+        return type(self.data[key])
+
+    def get_schema_for_key(self, key: str) -> dict:
+        """Get JSON schema for a specific key."""
+        if key not in self.data:
+            raise KeyError(f"Key '{key}' not found")
+
+        value = self.data[key]
+        if isinstance(value, str):
+            return {"type": "string"}
+        elif isinstance(value, bool):
+            return {"type": "boolean"}
+        elif isinstance(value, int):
+            return {"type": "integer"}
+        elif isinstance(value, float):
+            return {"type": "number"}
+        elif isinstance(value, datetime):
+            return {"type": "string", "format": "date-time"}
+        else:
+            return {"type": "string"}

simstack/models/file_instance.py

@@ -0,0 +1,77 @@
+import logging
+import shutil
+from datetime import datetime
+from pathlib import Path
+from typing import Union
+
+from odmantic import EmbeddedModel, Field, ObjectId
+
+from simstack.models import simstack_model
+
+logger = logging.getLogger("file_instance")
+
+
+@simstack_model
+class FileInstance(EmbeddedModel):
+    """ """
+
+    path: str = Field(
+        description="Path to the file relative to the host work directory"
+    )
+    resource: str = Field(description="Resource name")
+
+    created_at: datetime = Field(description="Creation timestamp")
+
+    @classmethod
+    def from_local_file(
+        cls, path: Union[Path, str], file_stack_id: ObjectId, make_copy: bool = True
+    ):
+        """
+        Creates a FileInstance object from a local file path.
+
+        This class method is responsible for initializing a `FileInstance` based
+        on a local file's path. It supports options to hash the file, make a
+        user-specific copy, and tracks additional metadata. The method handles
+        local file operations such as copying files to a secure directory when
+        necessary and organizes resources under a configurable working directory.
+
+        :param file_stack_id: the id of the filestack where the file is in
+        :param path: The file path to the local file. Can be either a string
+            or `Path`.
+        :param make_copy: Indicates whether a secure local copy of the file should
+            be made within the application's working directory. Defaults to True.
+        :return: A `FileInstance` object initialized with file details.
+        :rtype: FileInstance
+        :raises ValueError: If there are issues during the creation of the
+            `FileInstance` from the specified local file.
+        """
+        source_path = path if isinstance(path, Path) else Path(path)
+
+        # Prepare the content field if in_memory is True
+
+        try:
+            from simstack.core.context import context
+
+            if make_copy:
+                # make a local copy
+                import getpass
+
+                username = getpass.getuser()
+                relative_path = Path(username) / str(file_stack_id)
+                absolute_dir = Path(context.config.workdir) / relative_path
+                absolute_dir.mkdir(parents=True, exist_ok=True)
+                shutil.copy(path, absolute_dir)
+            else:
+                relative_path = Path(path).relative_to(context.config.workdir)
+
+            file_instance = FileInstance(
+                path=str(relative_path / source_path.name),
+                resource=context.config.resource,
+                created_at=datetime.now(),
+            )
+            return file_instance
+        except Exception as e:
+            logger.error(f"Error creating FileInstance from local file {path}: {e}")
+            raise ValueError(
+                f"Could not create FileInstance from local file {path}: {e}"
+            )

simstack/models/file_list.py

@@ -0,0 +1,321 @@
+import re
+from typing import Union, List, Any, Optional
+
+from odmantic import EmbeddedModel, Field, Model
+
+from simstack.models import simstack_model
+from simstack.models.files import FileStack
+
+
+class FileListMixin:
+    """
+    Mixin class containing common functionality for file list operations.
+    """
+
+    def __len__(self):
+        """
+        Returns the number of file stacks in the file list.
+
+        :return: Number of file stacks
+        :rtype: int
+        """
+        return len(self.file_stacks)
+
+    def append(self, file_stack: FileStack):
+        """
+        Appends a FileStack to the file list.
+
+        :param file_stack: The FileStack to append.
+        :type file_stack: FileStack
+        """
+        self.file_stacks.append(file_stack)
+
+    def extend(self, file_stacks: Union[List[FileStack], "FileListMixin"]):
+        """
+        Extends the file list with multiple FileStacks.
+
+        :param file_stacks: List of FileStacks or another FileListMixin to extend with
+        :type file_stacks: Union[List[FileStack], FileListMixin]
+        """
+        if hasattr(file_stacks, "file_stacks"):
+            # It's another FileListMixin object
+            self.file_stacks.extend(file_stacks.file_stacks)
+        else:
+            # It's a list
+            self.file_stacks.extend(file_stacks)
+
+    def insert(self, index: int, file_stack: FileStack):
+        """
+        Inserts a FileStack at the specified index.
+
+        :param index: Index where to insert the FileStack
+        :type index: int
+        :param file_stack: The FileStack to insert
+        :type file_stack: FileStack
+        """
+        self.file_stacks.insert(index, file_stack)
+
+    def remove(self, file_stack: FileStack):
+        """
+        Removes the first occurrence of the specified FileStack.
+
+        :param file_stack: The FileStack to remove
+        :type file_stack: FileStack
+        :raises ValueError: If the FileStack is not found
+        """
+        self.file_stacks.remove(file_stack)
+
+    def pop(self, index: int = -1) -> FileStack:
+        """
+        Removes and returns the FileStack at the specified index.
+
+        :param index: Index of the FileStack to remove (default is last item)
+        :type index: int
+        :return: The removed FileStack
+        :rtype: FileStack
+        :raises IndexError: If the index is out of range
+        """
+        return self.file_stacks.pop(index)
+
+    def clear(self):
+        """
+        Removes all FileStacks from the file list.
+        """
+        self.file_stacks.clear()
+
+    def index(self, file_stack: FileStack, start: int = 0, stop: int = None) -> int:
+        """
+        Returns the index of the first occurrence of the specified FileStack.
+
+        :param file_stack: The FileStack to find
+        :type file_stack: FileStack
+        :param start: Start index for search
+        :type start: int
+        :param stop: Stop index for search
+        :type stop: int
+        :return: Index of the FileStack
+        :rtype: int
+        :raises ValueError: If the FileStack is not found
+        """
+        if stop is None:
+            return self.file_stacks.index(file_stack, start)
+        return self.file_stacks.index(file_stack, start, stop)
+
+    def count(self, file_stack: FileStack) -> int:
+        """
+        Returns the number of occurrences of the specified FileStack.
+
+        :param file_stack: The FileStack to count
+        :type file_stack: FileStack
+        :return: Number of occurrences
+        :rtype: int
+        """
+        return self.file_stacks.count(file_stack)
+
+    def reverse(self):
+        """
+        Reverses the order of FileStacks in the file list.
+        """
+        self.file_stacks.reverse()
+
+    def sort(self, key=None, reverse: bool = False):
+        """
+        Sorts the FileStacks in the file list.
+
+        :param key: Function to extract comparison key from each FileStack
+        :param reverse: If True, sort in descending order
+        :type reverse: bool
+        """
+        self.file_stacks.sort(key=key, reverse=reverse)
+
+    def copy(self) -> List[FileStack]:
+        """
+        Returns a shallow copy of the file stacks list.
+
+        :return: Copy of the file stacks list
+        :rtype: List[FileStack]
+        """
+        return self.file_stacks.copy()
+
+    def __getitem__(
+        self, index: Union[int, slice]
+    ) -> Union[FileStack, List[FileStack]]:
+        """
+        Gets FileStack(s) at the specified index or slice.
+
+        :param index: Index or slice
+        :type index: Union[int, slice]
+        :return: FileStack or list of FileStacks
+        :rtype: Union[FileStack, List[FileStack]]
+        """
+        return self.file_stacks[index]
+
+    def __setitem__(
+        self, index: Union[int, slice], value: Union[FileStack, List[FileStack]]
+    ):
+        """
+        Sets FileStack(s) at the specified index or slice.
+
+        :param index: Index or slice
+        :type index: Union[int, slice]
+        :param value: FileStack or list of FileStacks to set
+        :type value: Union[FileStack, List[FileStack]]
+        """
+        self.file_stacks[index] = value
+
+    def __delitem__(self, index: Union[int, slice]):
+        """
+        Deletes FileStack(s) at the specified index or slice.
+
+        :param index: Index or slice
+        :type index: Union[int, slice]
+        """
+        del self.file_stacks[index]
+
+    def __iter__(self):
+        """
+        Returns an iterator over the FileStacks.
+
+        :return: Iterator over FileStacks
+        """
+        return iter(self.file_stacks)
+
+    def __contains__(self, file_stack: FileStack) -> bool:
+        """
+        Checks if a FileStack is in the file list.
+
+        :param file_stack: The FileStack to check for
+        :type file_stack: FileStack
+        :return: True if the FileStack is in the list, False otherwise
+        :rtype: bool
+        """
+        return file_stack in self.file_stacks
+
+    def __bool__(self) -> bool:
+        """
+        Returns True if the file list is not empty, False otherwise.
+
+        :return: True if file list has items, False if empty
+        :rtype: bool
+        """
+        return bool(self.file_stacks)
+
+    def __repr__(self) -> str:
+        """
+        Returns a string representation of the file list.
+
+        :return: String representation
+        :rtype: str
+        """
+        return f"{self.__class__.__name__}(file_stacks={self.file_stacks!r})"
+
+    def find(self, pattern: str) -> Union[FileStack, None]:
+        """
+        Searches through file stacks for the first file name matching the given regex pattern.
+
+        :param pattern: Regular expression pattern to match against file names
+        :type pattern: str
+        :return: First matching FileStack or None if no match found
+        :rtype: Union[FileStack, None]
+        """
+        for file_stack in self.file_stacks:
+            if file_stack.name and re.search(pattern, file_stack.name):
+                return file_stack
+        return None
+
+    def find_all(self, pattern: str) -> List[FileStack]:
+        """
+        Searches through file stacks for all files with names matching the given regex pattern.
+
+        :param pattern: Regular expression pattern to match against file names
+        :type pattern: str
+        :return: List of all matching FileStacks
+        :rtype: List[FileStack]
+        """
+        matches = []
+        for file_stack in self.file_stacks:
+            if file_stack.name and re.search(pattern, file_stack.name):
+                matches.append(file_stack)
+        return matches
+
+    def filter_by_size(
+        self, min_size: int = None, max_size: int = None
+    ) -> List[FileStack]:
+        """
+        Filters FileStacks by size range.
+
+        :param min_size: Minimum file size (inclusive)
+        :type min_size: int
+        :param max_size: Maximum file size (inclusive)
+        :type max_size: int
+        :return: List of FileStacks within the size range
+        :rtype: List[FileStack]
+        """
+        results = []
+        for file_stack in self.file_stacks:
+            if min_size is not None and file_stack.size < min_size:
+                continue
+            if max_size is not None and file_stack.size > max_size:
+                continue
+            results.append(file_stack)
+        return results
+
+    def filter_by_property(self, property_name: str, value: Any) -> List[FileStack]:
+        """
+        Filters FileStacks by a specific property value.
+
+        :param property_name: Name of the property to filter by
+        :type property_name: str
+        :param value: Value to match
+        :type value: Any
+        :return: List of matching FileStacks
+        :rtype: List[FileStack]
+        """
+        results = []
+        for file_stack in self.file_stacks:
+            if (
+                hasattr(file_stack, property_name)
+                and getattr(file_stack, property_name) == value
+            ):
+                results.append(file_stack)
+        return results
+
+    def sort_by_name(self, reverse: bool = False):
+        """
+        Sorts FileStacks by name.
+
+        :param reverse: If True, sort in descending order
+        :type reverse: bool
+        """
+        self.file_stacks.sort(key=lambda fs: fs.name or "", reverse=reverse)
+
+    def sort_by_size(self, reverse: bool = False):
+        """
+        Sorts FileStacks by size.
+
+        :param reverse: If True, sort in descending order
+        :type reverse: bool
+        """
+        self.file_stacks.sort(key=lambda fs: fs.size, reverse=reverse)
+
+
+@simstack_model
+class FileList(EmbeddedModel, FileListMixin):
+    file_stacks: List[FileStack] = Field(
+        default_factory=list, description="List of file stacks"
+    )
+
+
+@simstack_model
+class FileListModel(Model, FileListMixin):
+    file_stacks: List[FileStack] = Field(
+        default_factory=list, description="List of file stacks"
+    )
+
+
+@simstack_model
+class FileListIO(Model):
+    file_list: FileList = Field(default_factory=FileList)
+    task_status: Optional[str] = None
+    error: Optional[str] = None
+    message: Optional[str] = None