simstack 0.2.0__py3-none-any.whl

simstack/util/directory_iterator.py

@@ -0,0 +1,216 @@
+import fnmatch
+import os
+from pathlib import Path
+from typing import Iterator, List, Union, Optional
+
+
+class DirectoryPath:
+    """
+    Represents a directory path with excluded subdirectory patterns and provides iteration capabilities.
+    """
+
+    def __init__(
+        self,
+        root_path: Union[str, Path],
+        excluded_patterns: Optional[List[str]] = None,
+        file_extensions: Optional[List[str]] = None,
+    ):
+        """
+        Initialize the DirectoryPath.
+
+        Args:
+            root_path: The root directory path to traverse
+            excluded_patterns: List of patterns to exclude (supports wildcards like *, ?, [])
+            file_extensions: List of file extensions to include (e.g., ['.py', '.txt'])
+        """
+        self.root_path = Path(root_path).resolve()
+        self.excluded_patterns = excluded_patterns or []
+        self.file_extensions = file_extensions or []
+
+        if not self.root_path.exists():
+            raise FileNotFoundError(f"Directory does not exist: {self.root_path}")
+        if not self.root_path.is_dir():
+            raise NotADirectoryError(f"Path is not a directory: {self.root_path}")
+
+    def _is_excluded(self, path: Path) -> bool:
+        """
+        Check if a path matches any of the excluded patterns.
+
+        Args:
+            path: Path to check
+
+        Returns:
+            True if the path should be excluded, False otherwise
+        """
+        path_str = str(path)
+        relative_path = str(path.relative_to(self.root_path))
+
+        for pattern in self.excluded_patterns:
+            # Check against full path
+            if fnmatch.fnmatch(path_str, pattern):
+                return True
+            # Check against relative path
+            if fnmatch.fnmatch(relative_path, pattern):
+                return True
+            # Check against directory name only
+            if fnmatch.fnmatch(path.name, pattern):
+                return True
+
+        return False
+
+    def _matches_extensions(self, file_path: Path) -> bool:
+        """
+        Check if a file matches the specified extensions.
+
+        Args:
+            file_path: File path to check
+
+        Returns:
+            True if the file matches the extensions or no extensions specified
+        """
+        if not self.file_extensions:
+            return True
+
+        return file_path.suffix.lower() in [ext.lower() for ext in self.file_extensions]
+
+    def iterate_files(self) -> Iterator[Path]:
+        """
+        Iterate over all files in the directory tree, respecting exclusion patterns.
+
+        Yields:
+            Path objects for each file found
+        """
+        for root, dirs, files in os.walk(self.root_path):
+            root_path = Path(root)
+
+            # Filter out excluded directories in-place to prevent walking into them
+            dirs[:] = [d for d in dirs if not self._is_excluded(root_path / d)]
+
+            # Yield files that match criteria
+            for file in files:
+                file_path = root_path / file
+                if not self._is_excluded(file_path) and self._matches_extensions(
+                    file_path
+                ):
+                    yield file_path
+
+    def iterate_directories(self) -> Iterator[Path]:
+        """
+        Iterate over all directories in the directory tree, respecting exclusion patterns.
+
+        Yields:
+            Path objects for each directory found
+        """
+        for root, dirs, _ in os.walk(self.root_path):
+            root_path = Path(root)
+
+            # Filter and yield directories
+            for dir_name in dirs:
+                dir_path = root_path / dir_name
+                if not self._is_excluded(dir_path):
+                    yield dir_path
+
+            # Filter out excluded directories to prevent walking into them
+            dirs[:] = [d for d in dirs if not self._is_excluded(root_path / d)]
+
+    def __iter__(self) -> Iterator[Path]:
+        """
+        Default iterator returns files.
+        """
+        return self.iterate_files()
+
+    def get_files_list(self) -> List[Path]:
+        """
+        Get a list of all files matching the criteria.
+
+        Returns:
+            List of Path objects
+        """
+        return list(self.iterate_files())
+
+    def get_directories_list(self) -> List[Path]:
+        """
+        Get a list of all directories matching the criteria.
+
+        Returns:
+            List of Path objects
+        """
+        return list(self.iterate_directories())
+
+    def add_excluded_pattern(self, pattern: str) -> None:
+        """
+        Add an exclusion pattern.
+
+        Args:
+            pattern: Pattern to exclude
+        """
+        if pattern not in self.excluded_patterns:
+            self.excluded_patterns.append(pattern)
+
+    def remove_excluded_pattern(self, pattern: str) -> None:
+        """
+        Remove an exclusion pattern.
+
+        Args:
+            pattern: Pattern to remove
+        """
+        if pattern in self.excluded_patterns:
+            self.excluded_patterns.remove(pattern)
+
+    def __str__(self) -> str:
+        return (
+            f"DirectoryPath(root='{self.root_path}', excluded={self.excluded_patterns})"
+        )
+
+    def __repr__(self) -> str:
+        return (
+            f"DirectoryPath(root_path='{self.root_path}', "
+            f"excluded_patterns={self.excluded_patterns}, "
+            f"file_extensions={self.file_extensions})"
+        )
+
+
+# Example usage and utility functions
+def find_python_files_with_exclusions(
+    root_path: str, excluded_patterns: List[str] = None
+) -> List[Path]:
+    """
+    Find Python files using the DirectoryPath class with exclusions.
+
+    Args:
+        root_path: Root directory to search
+        excluded_patterns: Patterns to exclude
+
+    Returns:
+        List of Python file paths
+    """
+    excluded_patterns = excluded_patterns or [
+        "__pycache__",
+        "*.pyc",
+        ".git",
+        ".venv",
+        "venv",
+    ]
+    dir_path = DirectoryPath(root_path, excluded_patterns, [".py"])
+    return dir_path.get_files_list()
+
+
+if __name__ == "__main__":
+    # Example usage
+    try:
+        # Create a DirectoryPath instance
+        dp = DirectoryPath(
+            root_path=".",
+            excluded_patterns=["__pycache__", "*.pyc", ".git", "node_modules", "*.log"],
+            file_extensions=[".py", ".txt"],
+        )
+
+        print(f"Directory Path: {dp}")
+        print("\nFound files:")
+        for file_path in dp:
+            print(f"  {file_path}")
+
+        print(f"\nTotal files found: {len(dp.get_files_list())}")
+
+    except (FileNotFoundError, NotADirectoryError) as e:
+        print(f"Error: {e}")

simstack/util/file_hashing.py

@@ -0,0 +1,68 @@
+import hashlib
+from pathlib import Path
+from typing import Union, BinaryIO
+
+def hash_file(
+    file_path: Union[str, Path], algorithm: str = "sha256", chunk_size: int = 8192
+) -> str:
+    """
+    Calculate hash of a file on disk.
+
+    Args:
+        file_path: Path to the file
+        algorithm: Hash algorithm (md5, sha1, sha256, etc.)
+        chunk_size: Size of chunks to read
+
+    Returns:
+        str: Hexadecimal digest of the hash
+    """
+    hash_obj = hashlib.new(algorithm)
+
+    with open(file_path, "rb") as f:
+        for chunk in iter(lambda: f.read(chunk_size), b""):
+            hash_obj.update(chunk)
+
+    return hash_obj.hexdigest()
+
+
+def hash_file_object(
+    file_obj: BinaryIO, algorithm: str = "sha256", chunk_size: int = 8192
+) -> str:
+    """
+    Calculate hash of a file-like object.
+
+    Args:
+        file_obj: File-like object (must be in binary mode)
+        algorithm: Hash algorithm (md5, sha1, sha256, etc.)
+        chunk_size: Size of chunks to read
+
+    Returns:
+        str: Hexadecimal digest of the hash
+    """
+    hash_obj = hashlib.new(algorithm)
+
+    # Store current position
+    try:
+        original_position = file_obj.tell()
+    except (OSError, IOError, AttributeError):
+        original_position = None
+
+    # Seek to the beginning if possible
+    try:
+        file_obj.seek(0)
+    except (OSError, IOError, AttributeError):
+        pass
+
+    # Calculate hash
+    for chunk in iter(lambda: file_obj.read(chunk_size), b""):
+        hash_obj.update(chunk)
+
+    # Restore original position if possible
+    if original_position is not None:
+        try:
+            file_obj.seek(original_position)
+        except (OSError, IOError, AttributeError):
+            pass
+
+    return hash_obj.hexdigest()
+

simstack/util/files.py

@@ -0,0 +1,396 @@
+import os
+from datetime import datetime
+from typing import Optional, Dict, Any, List
+
+from odmantic import Field
+from pydantic import BaseModel
+
+
+class SshAuthMethod(BaseModel):
+    """ """
+
+    # Authentication type
+    auth_type: str = Field(
+        description="Authentication type (password, key, agent, none)"
+    )
+
+    # Password authentication
+    password: Optional[str] = Field(
+        default=None,
+        description="Password for authentication (should be stored securely)",
+    )
+
+    # Key-based authentication
+    key_filename: Optional[str] = Field(
+        default=None, description="Path to the private key file"
+    )
+    key_passphrase: Optional[str] = Field(
+        default=None, description="Passphrase for the private key"
+    )
+
+    # Allow host key to be added automatically to known_hosts
+    allow_host_key_add: bool = Field(
+        default=True, description="Whether to automatically add host key to known_hosts"
+    )
+
+    # Host key verification options
+    host_key_policy: str = Field(
+        default="ask", description="Host key policy (ask, auto_add, strict)"
+    )
+    known_hosts_file: Optional[str] = Field(
+        default=None, description="Path to known_hosts file"
+    )
+
+    model_config = {"extra": "allow"}
+
+
+class FfspecFile(BaseModel):
+    """
+    Representation of a file with protocol-specific details, metadata, and utility methods
+    for remote or local operations.
+
+    This class facilitates working with files across different storage protocols such as
+    local filesystem, SFTP, SCP, or cloud-based protocols. It provides attributes for
+    authenticating, accessing, and manipulating files as well as methods for operations
+    like copying files, creating URIs, and testing connections.
+
+    :ivar path: Full path to the file.
+    :type path: str
+    :ivar name: Name of the file.
+    :type name: str
+    :ivar extension: File extension.
+    :type extension: str
+    :ivar protocol: Storage protocol (e.g., 'file', 's3', 'http', 'sftp', 'scp').
+    :type protocol: str
+    :ivar host: Host name or address where the file is located.
+    :type host: Optional[str]
+    :ivar port: Port number if applicable.
+    :type port: Optional[int]
+    :ivar username: Username for authentication if required.
+    :type username: Optional[str]
+    :ivar ssh_auth: SSH authentication configuration.
+    :type ssh_auth: Optional[SshAuthMethod]
+    :ivar size: Size of the file in bytes.
+    :type size: int
+    :ivar created_at: Creation timestamp.
+    :type created_at: datetime
+    :ivar modified_at: Last modification timestamp.
+    :type modified_at: datetime
+    :ivar storage_options: Protocol-specific storage options.
+    :type storage_options: Dict[str, Any]
+    :ivar checksum: File content checksum.
+    :type checksum: Optional[str]
+    :ivar content_type: MIME type of the file.
+    :type content_type: Optional[str]
+    :ivar parent_path: Path to parent directory.
+    :type parent_path: Optional[str]
+    :ivar is_directory: Whether this is a directory.
+    :type is_directory: bool
+    :ivar children: Child file/directory paths if this is a directory.
+    :type children: List[str]
+    :ivar last_accessed: Last access timestamp.
+    :type last_accessed: Optional[datetime]
+    :ivar access_count: Number of times this file has been accessed.
+    :type access_count: int
+    :ivar metadata: Additional metadata as key-value pairs.
+    :type metadata: Dict[str, Any]
+    """
+
+    # Basic file information
+    path: str = Field(description="Full path to the file")
+    name: str = Field(description="Name of the file")
+    extension: str = Field(description="File extension")
+
+    # Protocol and location information
+    protocol: str = Field(
+        description="Storage protocol (e.g., 'file', 's3', 'http', 'sftp', 'scp')"
+    )
+    host: Optional[str] = Field(
+        default=None, description="Host name or address where the file is located"
+    )
+    port: Optional[int] = Field(default=None, description="Port number if applicable")
+
+    # Authentication information
+    username: Optional[str] = Field(
+        default=None, description="Username for authentication if required"
+    )
+    ssh_auth: Optional[SshAuthMethod] = Field(
+        default=None, description="SSH authentication configuration"
+    )
+
+    # File metadata
+    size: int = Field(description="Size of the file in bytes")
+    created_at: datetime = Field(description="Creation timestamp")
+    modified_at: datetime = Field(description="Last modification timestamp")
+
+    # Ffspec-specific attributes
+    storage_options: Dict[str, Any] = Field(
+        default_factory=dict, description="Protocol-specific storage options"
+    )
+
+    # Content addressing
+    checksum: Optional[str] = Field(default=None, description="File content checksum")
+    content_type: Optional[str] = Field(
+        default=None, description="MIME type of the file"
+    )
+
+    # Hierarchical structure support
+    parent_path: Optional[str] = Field(
+        default=None, description="Path to parent directory"
+    )
+    is_directory: bool = Field(default=False, description="Whether this is a directory")
+    children: List[str] = Field(
+        default_factory=list,
+        description="Child file/directory paths if this is a directory",
+    )
+
+    # Access metadata
+    last_accessed: Optional[datetime] = Field(
+        default=None, description="Last access timestamp"
+    )
+    access_count: int = Field(
+        default=0, description="Number of times this file has been accessed"
+    )
+
+    # Custom metadata
+    metadata: Dict[str, Any] = Field(
+        default_factory=dict, description="Additional metadata as key-value pairs"
+    )
+
+    @classmethod
+    def from_ssh_key(
+        cls,
+        host: str,
+        path: str,
+        username: str,
+        key_filename: str,
+        protocol: str = "sftp",
+        port: int = 22,
+        key_passphrase: Optional[str] = None,
+        host_key_policy: str = "auto_add",
+        **kwargs,
+    ) -> "FfspecFile":
+        """
+        Create a FfspecFile instance with SSH key authentication.
+
+        Args:
+            host: Remote host name or IP address
+            path: Path to the file on the remote host
+            username: SSH username for authentication
+            key_filename: Path to the SSH private key file
+            protocol: Protocol to use, either 'sftp' or 'scp' (default: 'sftp')
+            port: SSH port on the remote host (default: 22)
+            key_passphrase: Passphrase for the SSH key if it's encrypted (default: None)
+            host_key_policy: Host key policy - 'auto_add', 'strict', or 'ask' (default: 'auto_add')
+            **kwargs: Additional attributes to set on the model
+
+        Returns:
+            FfspecFile: A new instance representing the remote file
+        """
+        from pathlib import Path
+
+        # Validate protocol
+        if protocol not in ["sftp", "scp"]:
+            raise ValueError(f"Protocol must be 'sftp' or 'scp', got '{protocol}'")
+
+        # Validate that the key file exists
+        if not os.path.exists(key_filename):
+            raise FileNotFoundError(f"SSH key file not found: {key_filename}")
+
+        # Ensure path is absolute (starts with /)
+        if not path.startswith("/"):
+            path = "/" + path
+
+        # Create SSH authentication config with proper host key policy
+        ssh_auth = SshAuthMethod(
+            auth_type="key",
+            key_filename=key_filename,
+            key_passphrase=key_passphrase,
+            host_key_policy=host_key_policy,
+            allow_host_key_add=(host_key_policy == "auto_add"),
+        )
+
+        # Extract file name and extension from path
+        p = Path(path)
+        name = p.name
+        extension = p.suffix.lstrip(".") if p.suffix else ""
+        parent_path = str(p.parent) if p.parent != p else None
+
+        # Create comprehensive storage options for fsspec
+        # Note: Removed missing_host_key_policy as it's not a valid paramiko connection parameter
+        storage_options = {
+            "host": host,
+            "port": port,
+            "username": username,
+            "key_filename": key_filename,
+            # Connection timeouts
+            "timeout": 30,
+            "banner_timeout": 30,
+            "auth_timeout": 30,
+        }
+
+        # Add key passphrase if provided
+        if key_passphrase:
+            storage_options["password"] = key_passphrase
+
+        # Create the instance
+        return cls(
+            path=path,
+            name=name,
+            extension=extension,
+            protocol=protocol,
+            host=host,
+            port=port,
+            username=username,
+            ssh_auth=ssh_auth,
+            storage_options=storage_options,
+            parent_path=parent_path,
+            # Use values from kwargs or sensible defaults for required fields
+            size=kwargs.pop("size", 0),
+            created_at=kwargs.pop("created_at", datetime.now()),
+            modified_at=kwargs.pop("modified_at", datetime.now()),
+            is_directory=kwargs.pop("is_directory", False),
+            **kwargs,
+        )
+
+    def get_filesystem(self):
+        """
+        Create and return a fsspec filesystem object for this file.
+
+        Returns:
+            object: An fsspec filesystem instance
+        """
+        import importlib
+
+        try:
+            # Import the appropriate filesystem module
+            fsspec = importlib.import_module("fsspec")
+
+            # Handle SSH-specific protocols
+            if self.protocol in ["sftp", "scp"]:
+                # Create a copy of storage options to avoid modifying the original
+                storage_options = dict(self.storage_options)
+
+                # Add SSH auth options if available
+                if self.ssh_auth:
+                    if self.ssh_auth.auth_type == "key":
+                        storage_options["key_filename"] = self.ssh_auth.key_filename
+                        if self.ssh_auth.key_passphrase:
+                            storage_options["password"] = self.ssh_auth.key_passphrase
+                    elif self.ssh_auth.auth_type == "password":
+                        storage_options["password"] = self.ssh_auth.password
+
+                # Create the filesystem without the problematic missing_host_key_policy parameter
+                return fsspec.filesystem(self.protocol, **storage_options)
+            else:
+                # Handle other protocols
+                return fsspec.filesystem(self.protocol, **self.storage_options)
+
+        except (ImportError, KeyError) as e:
+            raise ValueError(
+                f"Could not create filesystem for protocol '{self.protocol}': {e}"
+            )
+
+    def can_connect(self) -> bool:
+        """
+        Test if we can connect to the remote host using the provided credentials.
+
+        Returns:
+            bool: True if connection is successful, False otherwise
+        """
+        if self.protocol not in ["sftp", "scp"]:
+            # Only applicable for SSH-based protocols
+            return False
+
+        try:
+            fs = self.get_filesystem()
+            # Try to list the parent directory
+            fs.ls(os.path.dirname(self.path))
+            return True
+        except Exception:
+            return False
+
+    def to_uri(self) -> str:
+        """
+        Convert this model to a URI/URL representation.
+
+        Note: SSH private key information is not included in the URI.
+
+        Returns:
+            str: URI string representing this file
+        """
+        import urllib.parse
+
+        # Build the netloc part
+        netloc = ""
+        if self.username:
+            netloc = f"{self.username}@"
+        if self.host:
+            netloc += self.host
+        if self.port and self.port != 22:  # Only include non-standard SSH port
+            netloc += f":{self.port}"
+
+        # Assemble the URI
+        return urllib.parse.urlunparse(
+            (
+                self.protocol,  # scheme
+                netloc,  # netloc
+                self.path,  # path
+                "",  # params
+                "",  # query (not including auth info for security)
+                "",  # fragment
+            )
+        )
+
+    def copy_to_local(self, local_path: Optional[str] = None) -> str:
+        """
+        Copy the remote file to a local path.
+
+        Args:
+            local_path: Local path to copy the file to. If None, a temporary file is created.
+
+        Returns:
+            str: Path to the local copy of the file
+        """
+        import tempfile
+
+        if self.is_directory:
+            raise ValueError(
+                "Cannot copy a directory, only individual files are supported"
+            )
+
+        # Create a destination path if none is provided
+        if local_path is None:
+            temp_dir = tempfile.gettempdir()
+            local_path = os.path.join(temp_dir, self.name)
+
+        # Get the filesystem and copy the file
+        fs = self.get_filesystem()
+        fs.get(self.path, local_path)
+
+        return local_path
+
+    def copy_from_local(self, local_path: str) -> None:
+        """
+        Copy a local file to the remote location represented by this model.
+
+        Args:
+            local_path: Path to the local file
+
+        Returns:
+            None
+        """
+        if not os.path.exists(local_path):
+            raise FileNotFoundError(f"Local file {local_path} does not exist")
+
+        # Get the filesystem and copy the file
+        fs = self.get_filesystem()
+        fs.put(local_path, self.path)
+
+        # Update file size and modification time
+        try:
+            info = fs.info(self.path)
+            self.size = info.get("size", 0)
+            self.modified_at = datetime.now()
+        except Exception:
+            pass