eoml 0.9.0__py3-none-any.whl

eoml/__init__.py

@@ -0,0 +1,74 @@
+"""
+Earth Observation Machine Learning (EOML) Package.
+
+This package provides tools and utilities for processing Earth observation data
+and machine learning workflows for remote sensing applications. It includes modules
+for data processing, raster operations, neural network training, and automation.
+
+The package is organized into the following main modules:
+- automation: Task automation and experiment configuration
+- data: Data structures and persistence utilities
+- ee: Google Earth Engine integration
+- raster: Raster data reading and processing
+- torch: PyTorch-based machine learning models and training
+- bin: Command-line utilities
+"""
+
+default_read_profile = {'num_threads': 'all_cpus'}  #'all_cpus'
+
+default_write_profile = {'driver': 'GTiff',
+                        'BIGTIFF':  'IF_SAFER',
+                        'num_threads': 'all_cpus',
+                        'tiled': True,
+                        'blockxsize': 512,
+                        'blockysize': 512,
+                        'compress': 'zstd'}
+
+
+def get_read_profile(**kargs):
+    """
+    Get a default reasonable TIFF reader profile.
+
+    Returns a profile dictionary with default settings for reading GeoTIFF files
+    using rasterio. The default profile uses all available CPU threads for reading.
+
+    Args:
+        **kargs: Additional keyword arguments to override default profile settings.
+
+    Returns:
+        dict: A profile dictionary suitable for use with rasterio.open().
+
+    Examples:
+        >>> profile = get_read_profile()
+        >>> profile = get_read_profile(num_threads=4)
+    """
+    profile = default_read_profile.copy()
+    profile.update(kargs)
+    return profile
+
+
+def get_write_profile(**kargs):
+    """
+    Get a default reasonable TIFF writer profile.
+
+    Returns a profile dictionary with default settings for writing GeoTIFF files
+    using rasterio. The default profile uses:
+    - GTiff driver
+    - ZSTD compression
+    - Tiled format with 512x512 blocks
+    - All available CPU threads
+    - BIGTIFF when safer
+
+    Args:
+        **kargs: Additional keyword arguments to override default profile settings.
+
+    Returns:
+        dict: A profile dictionary suitable for use with rasterio.open() in write mode.
+
+    Examples:
+        >>> profile = get_write_profile()
+        >>> profile = get_write_profile(compress='lzw', num_threads=4)
+    """
+    profile = default_write_profile.copy()
+    profile.update(kargs)
+    return profile

eoml/automation/__init__.py

@@ -0,0 +1,7 @@
+"""
+Automation Module for EOML.
+
+This module provides automation utilities for machine learning experiments
+in Earth observation applications. It includes configuration management,
+task scheduling, and experiment workflow orchestration.
+"""

eoml/automation/configuration.py

@@ -0,0 +1,105 @@
+"""
+Configuration management module for EOML automation.
+
+This module provides system-wide configuration management for EOML experiments,
+including paths to data directories, raster processing profiles, and neural
+network device settings.
+"""
+
+import os
+
+import toml
+from typing import Any, Dict, Union, Optional
+
+from eoml import default_read_profile, default_write_profile
+
+# Pydantic model for configuration (v2)
+try:
+    from pydantic import BaseModel, Field, AliasChoices
+except Exception:  # pragma: no cover - optional import hint
+    BaseModel = object  # type: ignore
+    Field = lambda *args, **kwargs: None  # type: ignore
+    class AliasChoices:  # type: ignore
+        def __init__(self, *args, **kwargs):
+            pass
+
+SYSTEM_CONFIG = None
+
+class SystemConfigModel(BaseModel):
+    """
+    Pydantic representation of System configuration.
+
+    This class stores configuration settings for data directories, raster
+    processing profiles, and neural network execution parameters.
+
+    Attributes:
+        data_dir (str): Base directory for data storage.
+        raster_dir (str): Directory containing raster files.
+        shade_dir (str): Directory for shade/canopy data.
+        land_cover_dir (str): Directory for land cover data.
+        raster_read_profile (dict): Rasterio profile for reading rasters.
+        raster_write_profile (dict): Rasterio profile for writing rasters.
+        device (str): Device for neural network execution ('cpu', 'cuda', etc.).
+        mapping_mode (int): Mode for map generation (0=CPU, 1=GPU with pinned memory, etc.).
+    """
+
+    data_dir: str
+    raster_dir: str
+    shade_dir: str
+    land_cover_dir: str
+    raster_read_profile: Dict[str, Any]
+    raster_write_profile: Dict[str, Any]
+    device: str
+    mapping_mode: int
+
+    @classmethod
+    def load_toml(cls, path: str) -> "SystemConfigModel":
+        """Load a configuration model from a TOML file."""
+        data = toml.load(path)
+        return cls(**data)
+
+    def set_as_global(self, name: str = "default", set_default: bool = True) -> None:
+        """
+        Set this configuration as a global system configuration.
+
+        This method stores the configuration in the global SYSTEM_CONFIG dictionary
+        and optionally updates the default raster read/write profiles.
+
+        Args:
+            name (str, optional): Name to store this configuration under in the
+                SYSTEM_CONFIG dictionary. Defaults to "default".
+            set_default (bool, optional): If True, updates the global default_read_profile
+                and default_write_profile with values from this configuration.
+                Defaults to True.
+
+        Side Effects:
+            - Updates global SYSTEM_CONFIG dictionary
+            - Optionally updates global default_read_profile and default_write_profile
+
+        Examples:
+            >>> config = SystemConfigModel.load_toml("/path/to/config.toml")
+            >>> config.set_as_global("my_config")
+        """
+        global SYSTEM_CONFIG
+
+        SYSTEM_CONFIG[name] = self
+
+        if set_default:
+            default_read_profile.update(self.raster_read_profile)
+            default_write_profile.update(self.raster_write_profile)
+
+
+
+
+def get_config() -> Optional[SystemConfigModel]:
+    """
+    Retrieve a stored system configuration by name.
+
+    Returns:
+        SystemConfig: The requested configuration object.
+
+    Examples:
+        >>> config = get_config()
+    """
+    return SYSTEM_CONFIG
+

eoml/automation/dag.py

@@ -0,0 +1,233 @@
+"""
+Directed Acyclic Graph (DAG) module for task automation.
+
+This module provides classes for creating and managing directed acyclic graphs
+of tasks with dependencies. It enables workflow automation and task orchestration
+for machine learning experiments.
+"""
+
+
+class TaskGraph:
+    """
+    Container for a directed acyclic graph of tasks.
+
+    The TaskGraph manages a collection of tasks with dependencies, ensuring
+    unique task names and maintaining the execution order.
+
+    Attributes:
+        task_list (list): Ordered list of tasks in the graph.
+        task_dic (dict): Dictionary mapping task names to task objects.
+    """
+
+    def __init__(self, task_list):
+        """
+        Initialize a TaskGraph with a list of tasks.
+
+        Args:
+            task_list (list): List of Task objects to include in the graph.
+
+        Raises:
+            Exception: If duplicate task names are found.
+        """
+        self.task_list = task_list
+        self.task_dic = self._make_task_dic(task_list)
+
+        for t in task_list:
+            t.task_graph = self
+
+    def _make_task_dic(self, task_list):
+        """
+        Create a dictionary mapping task names to task objects.
+
+        Args:
+            task_list (list): List of Task objects.
+
+        Returns:
+            dict: Dictionary with task names as keys and Task objects as values.
+
+        Raises:
+            Exception: If duplicate task names are found.
+        """
+        task_dic = {}
+        for task in task_list:
+            if task.name in task_dic:
+                raise Exception(f"Duplicate task with task id {task.name}, task id should be unique ")
+
+            task_dic[task.name] = task
+
+        return task_dic
+
+    def manage_outputs(self, task_id, outputs):
+        """
+        Manage outputs of a task and propagate them to dependent tasks.
+
+        Args:
+            task_id: Identifier of the task producing outputs.
+            outputs: Outputs produced by the task.
+
+        Note:
+            This method is a placeholder and needs implementation.
+        """
+        # todo need function
+        index = self.task_list.index(task_id)
+
+        for n in range(index, len(self.task_list)):
+            pass
+
+
+class Task:
+    """
+    Base class representing a task in a task graph.
+
+    A task has a name, inputs, and can be executed as part of a workflow.
+    Tasks can depend on outputs from other tasks in the graph.
+
+    Attributes:
+        name (str): Unique identifier for the task.
+        inputs (list): List of TaskInput objects defining task inputs.
+        need (dict): Dictionary of dependencies on other tasks.
+        task_graph (TaskGraph): Reference to the containing task graph.
+    """
+
+    def __init__(self, name, inputs):
+        """
+        Initialize a Task.
+
+        Args:
+            name (str): Unique identifier for the task.
+            inputs (list): List of TaskInput objects defining inputs.
+        """
+        self.name = name
+        self.inputs = inputs
+
+        self.need = self._need()
+
+        self.task_graph: TaskGraph = None
+
+    def make_input(self):
+        """
+        Resolve task inputs from the task graph.
+
+        Returns:
+            dict: Dictionary of resolved input values, with input names as keys.
+        """
+        inputs_dic = {}
+
+        for task_input in self.inputs:
+
+            if task_input.depend:
+                inputs_dic[task_input.name] = self.task_graph.task_dic[task_input.value]
+            else:
+                inputs_dic[task_input.name] = task_input.value
+
+        return inputs_dic
+
+    def _need(self):
+        """
+        Compute the dependencies the task needs from other tasks.
+
+        Returns:
+            dict: Dictionary mapping dependency names to lists of input names.
+
+        Note:
+            This method needs implementation (TODO noted in original code).
+        """
+        # TODO
+
+        need = {}
+        for i in self.inputs:
+            if i.depend:
+                need.get(i.value, []).append(i.name)
+
+        return need
+
+    def execute(self):
+        """
+        Execute the task by resolving inputs and calling process().
+
+        This method resolves all inputs from the task graph and passes them
+        to the process() method for execution.
+        """
+        inputs = self.make_input()
+        self.process(**inputs)
+
+    def process(self, **inputs):
+        """
+        Process the task with resolved inputs.
+
+        This is a placeholder method to be overridden by subclasses to implement
+        specific task logic.
+
+        Args:
+            **inputs: Resolved input values as keyword arguments.
+        """
+        pass
+
+
+class TaskInput:
+    """
+    Represents an input to a task.
+
+    A TaskInput can either be a direct value or a dependency on another task's output.
+
+    Attributes:
+        name (str): Name of the input parameter.
+        value: The value or name of the dependency.
+        depend (bool): If True, value refers to another task name; if False, value is used directly.
+    """
+
+    def __init__(self, name, value, depend=False):
+        """
+        Initialize a TaskInput.
+
+        Args:
+            name (str): Name of the input parameter.
+            value: The value or name of the dependency.
+            depend (bool, optional): Whether this input depends on another task. Defaults to False.
+        """
+        self.name = name
+        self.value = value
+        self.depend = depend
+
+
+class Value(Task):
+    """
+    A task that represents a constant value.
+
+    This is a simple task type that holds a value without any inputs or processing.
+    Useful for providing constant parameters to other tasks in the graph.
+
+    Attributes:
+        value: The constant value held by this task.
+    """
+
+    def __init__(self, name, value):
+        """
+        Initialize a Value task.
+
+        Args:
+            name (str): Unique identifier for the task.
+            value: The constant value to hold.
+        """
+        super().__init__(name, [])
+        self.value = value
+
+
+class ReadProfile(Task):
+    """
+    A task for reading a profile or configuration.
+
+    This task is intended to read and provide configuration profiles to other tasks.
+
+    Note:
+        This appears to be a placeholder class requiring implementation.
+    """
+
+    def __init__(self, name):
+        """
+        Initialize a ReadProfile task.
+
+        Args:
+            name (str): Unique identifier for the task.
+        """
+        super().__init__(name, [])