hafnia 0.1.24__py3-none-any.whl → 0.1.26__py3-none-any.whl

hafnia/platform/executor.py

@@ -1,111 +0,0 @@
-import os
-import subprocess
-import sys
-from dataclasses import dataclass
-from pathlib import Path
-from typing import Dict
-
-from hafnia.log import logger
-
-
-@dataclass
-class PythonModule:
-    """Dataclass to store Python module details."""
-
-    module_name: str
-    runner_path: str
-
-
-def handle_mount(source: str) -> None:
-    """
-    Mounts the Hafnia environment by adding source directories to PYTHONPATH.
-
-    Args:
-        source (str): Path to the root directory containing 'lib' and 'scripts' subdirectories
-
-    Raises:
-        FileNotFoundError: If the required directory structure is not found
-    """
-    source_path = Path(source)
-    lib_dir = source_path / "lib"
-    scripts_dir = source_path / "scripts"
-
-    if not lib_dir.exists() and not scripts_dir.exists():
-        logger.error(f"Filestructure is not supported. Expected 'lib' and 'scripts' directories in {source_path}.")
-        exit(1)
-
-    sys.path.extend([lib_dir.as_posix(), scripts_dir.as_posix()])
-    python_path = os.getenv("PYTHONPATH", "")
-    os.environ["PYTHONPATH"] = f"{python_path}:{lib_dir.as_posix()}:{scripts_dir.as_posix()}"
-    logger.info(f"Mounted codebase from {source_path}")
-
-
-def collect_python_modules(directory: Path) -> Dict[str, PythonModule]:
-    """
-    Collects Python modules from a directory and its subdirectories.
-
-    This function dynamically imports Python modules found in the specified directory,
-    excluding files that start with '_' or '.'. It's used to discover available tasks
-    in the Hafnia environment.
-
-    Args:
-        directory (Path): The directory to search for Python modules
-
-    Returns:
-        Dict[str, Dict[str, str]]: A dictionary mapping task names to module details, where each detail contains:
-            - module_name (str): The full module name
-            - runner_path (str): The absolute path to the module file
-    """
-    from importlib.util import module_from_spec, spec_from_file_location
-
-    modules = {}
-    for fname in directory.rglob("*.py"):
-        if fname.name.startswith("-"):
-            continue
-
-        task_name = fname.stem
-        module_name = f"{directory.name}.{task_name}"
-
-        spec = spec_from_file_location(module_name, fname)
-        if spec is None:
-            logger.warning(f"Was not able to load {module_name} from {fname}")
-            continue
-        if spec.loader is None:
-            logger.warning(f"Loader is None for {module_name} from {fname}")
-            continue
-        module = module_from_spec(spec)
-        spec.loader.exec_module(module)
-
-        modules[task_name] = PythonModule(module_name, str(fname.resolve()))
-
-    return modules
-
-
-def handle_launch(task: str) -> None:
-    """
-    Launch and execute a specified Hafnia task.
-
-    This function verifies the Hafnia environment status, locates the task script,
-    and executes it in a subprocess with output streaming.
-
-    Args:
-        task (str): Name of the task to execute
-
-    Raises:
-        ValueError: If the task is not found or scripts directory is not in PYTHONPATH
-    """
-    recipe_dir = os.getenv("RECIPE_DIR", None)
-    if recipe_dir is None:
-        raise ValueError("RECIPE_DIR environment variable not set.")
-    handle_mount(recipe_dir)
-    scripts_dir = [p for p in sys.path if "scripts" in p][0]
-    scripts = collect_python_modules(Path(scripts_dir))
-    if task not in scripts:
-        available_tasks = ", ".join(sorted(scripts.keys()))
-        logger.error(f"Task '{task}' not found. Available tasks: {available_tasks}")
-        exit(1)
-    try:
-        subprocess.check_call(["python", scripts[task].runner_path], stdout=sys.stdout, stderr=sys.stdout)
-    except subprocess.CalledProcessError as e:
-        logger.error(f"Error executing task: {str(e)}")
-        exit(1)

hafnia-0.1.24.dist-info/METADATA

@@ -1,197 +0,0 @@
-Metadata-Version: 2.4
-Name: hafnia
-Version: 0.1.24
-Summary: Python tools for communication with Hafnia platform.
-Author-email: Ivan Sahumbaiev <ivsa@milestone.dk>
-License-File: LICENSE
-Requires-Python: >=3.10
-Requires-Dist: boto3>=1.35.91
-Requires-Dist: click>=8.1.8
-Requires-Dist: datasets>=3.2.0
-Requires-Dist: flatten-dict>=0.4.2
-Requires-Dist: pillow>=11.1.0
-Requires-Dist: pyarrow>=18.1.0
-Requires-Dist: pydantic>=2.10.4
-Requires-Dist: rich>=13.9.4
-Requires-Dist: tqdm>=4.67.1
-Provides-Extra: torch
-Requires-Dist: flatten-dict>=0.4.2; extra == 'torch'
-Requires-Dist: torch>=2.6.0; extra == 'torch'
-Requires-Dist: torchvision>=0.21.0; extra == 'torch'
-Description-Content-Type: text/markdown
-
-# Hafnia
-
-The `hafnia` python package is a collection of tools to create and run model training recipes on
-the [Hafnia Platform](https://hafnia.milestonesys.com/). 
-
-The package includes the following interfaces: 
-
-- `cli`: A Command Line Interface (CLI) to 1) configure/connect to Hafnia's [Training-aaS](https://hafnia.readme.io/docs/training-as-a-service) and 2) create and 
-launch recipe scripts.
-- `hafnia`: A python package with helper functions to load and interact with sample datasets and an experiment
- tracker (`HafniaLogger`). 
-
-
-## The Concept: Training as a Service (Training-aaS)
-`Training-aaS` is the concept of training models on the Hafnia platform on large 
-and *hidden* datasets. Hidden datasets refers to datasets that can be used for 
-training, but are not available for download or direct access. 
-
-This is a key feature of the Hafnia platform, as a hidden dataset ensures data 
-privacy, and allow models to be trained compliantly and ethically by third parties (you).
-
-The `script2model` approach is a Training-aaS concept, where you package your custom training 
-script as a *training recipe* and use the recipe to train models on the hidden datasets.
-
-To support local development of a training recipe, we have introduced a **sample dataset** 
-for each dataset available in the Hafnia [data library](https://hafnia.milestonesys.com/training-aas/datasets). The sample dataset is a small 
-and anonymized subset of the full dataset and available for download. 
-
-With the sample dataset, you can seamlessly switch between local development and Training-aaS. 
-Locally, you can create, validate and debug your training recipe. The recipe is then 
-launched with Training-aaS, where the recipe runs on the full dataset and can be scaled to run on
-multiple GPUs and instances if needed. 
-
-## Getting started: Configuration
-To get started with Hafnia: 
-
-1. Install `hafnia` with your favorite python package manager. With pip do this:
-
-    `pip install hafnia`
-1. Sign in to the [Hafnia Platform](https://hafnia.milestonesys.com/). 
-1. Create an API KEY for Training aaS. For more instructions, follow this 
-[guide](https://hafnia.readme.io/docs/create-an-api-key). 
-Copy the key and save it for later use.
-1. From terminal, configure your machine to access Hafnia: 
-
-    ```
-    # Start configuration with
-    hafnia configure
-
-    # You are then prompted: 
-    Profile Name [default]:   # Press [Enter] or select an optional name
-    Hafnia API Key:  # Pass your HAFNIA API key
-    Hafnia Platform URL [https://api.mdi.milestonesys.com]:  # Press [Enter]
-    ```
-1. Download `mnist` from terminal to verify that your configuration is working.  
-
-    ```bash
-    hafnia data download mnist --force
-    ```
-
-## Getting started: Loading datasets samples
-With Hafnia configured on your local machine, it is now possible to download 
-and explore the dataset sample with a python script:
-
-```python
-from hafnia.data import load_dataset
-
-dataset_splits = load_dataset("mnist")
-print(dataset_splits)
-print(dataset_splits["train"])
-```
-The returned sample dataset is a [hugging face dataset](https://huggingface.co/docs/datasets/index) 
-and contains train, validation and test splits. 
-
-An important feature of `load_dataset` is that it will return the full dataset 
-when loaded on the Hafnia platform. 
-This enables seamlessly switching between running/validating a training script 
-locally (on the sample dataset) and running full model trainings with Training-aaS (on the full dataset). 
-without changing code or configurations for the training script.
-
-Available datasets with corresponding sample datasets can be found in [data library](https://hafnia.milestonesys.com/training-aas/datasets) including metadata and description for each dataset. 
-
-
-## Getting started: Experiment Tracking with HafniaLogger
-The `HafniaLogger` is an important part of the recipe script and enables you to track, log and
-reproduce your experiments.
-
-When integrated into your training script, the `HafniaLogger` is responsible for collecting:
-
-- **Trained Model**: The model trained during the experiment
-- **Model Checkpoints**: Intermediate model states saved during training
-- **Experiment Configurations**: Hyperparameters and other settings used in your experiment
-- **Training/Evaluation Metrics**: Performance data such as loss values, accuracy, and custom metrics
-
-### Basic Implementation Example
-
-Here's how to integrate the `HafniaLogger` into your training script:
-
-```python
-from hafnia.experiment import HafniaLogger
-
-batch_size = 128
-learning_rate = 0.001
-
-# Initialize Hafnia logger
-logger = HafniaLogger()
-
-# Log experiment parameters
-logger.log_configuration({"batch_size": 128, "learning_rate": 0.001})
-
-# Store checkpoints in this path
-ckpt_dir = logger.path_model_checkpoints()
-
-# Store the trained model in this path
-model_dir = logger.path_model()
-
-# Log scalar and metric values during training and validation
-logger.log_scalar("train/loss", value=0.1, step=100)
-logger.log_metric("train/accuracy", value=0.98, step=100)
-
-logger.log_scalar("validation/loss", value=0.1, step=100)
-logger.log_metric("validation/accuracy", value=0.95, step=100)
-```
-
-Similar to `load_dataset`, the tracker behaves differently when running locally or in the cloud. 
-Locally, experiment data is stored in a local folder `.data/experiments/{DATE_TIME}`. 
-
-In the cloud, the experiment data will be available in the Hafnia platform under 
-[experiments](https://hafnia.milestonesys.com/training-aas/experiments). 
-
-## Example: Torch Dataloader
-Commonly for `torch`-based training scripts, a dataset is used in combination 
-with a dataloader that performs data augmentations and batching of the dataset as torch tensors.
-
-To support this, we have provided a torch dataloader example script
-[example_torchvision_dataloader.py](./examples/example_torchvision_dataloader.py). 
-
-The script demonstrates how to make a dataloader with data augmentation (`torchvision.transforms.v2`)
-and a helper function for visualizing image and labels. 
-
-The dataloader and visualization function supports computer vision tasks 
-and datasets available in the data library. 
-
-## Example: Training-aaS
-By combining logging and dataset loading, we can now construct our model training recipe. 
-
-To demonstrate this, we have provided a recipe project that serves as a template for creating and structuring training recipes
-[recipe-classification](https://github.com/Data-insight-Platform/recipe-classification)
-
-The project also contains additional information on how to structure your training recipe, use the `HafniaLogger`, the `load_dataset` function and different approach for launching 
-the training recipe on the Hafnia platform.
-
-## Detailed Documentation
-For more information, go to our [documentation page](https://hafnia.readme.io/docs/welcome-to-hafnia) 
-or in below markdown pages. 
-
-- [CLI](docs/cli.md) - Detailed guide for the Hafnia command-line interface
-- [Script2Model Documentation](docs/s2m.md) - Detailed guide for script2model
-- [Release lifecycle](docs/release.md) - Details about package release lifecycle.
-
-## Development
-For development, we are using an uv based virtual python environment.
-
-Install uv
-
-    curl -LsSf https://astral.sh/uv/install.sh | sh
-
-
-Install python dependencies including developer (`--dev`) and optional dependencies (`--all-extras`).
-
-    uv sync --all-extras --dev
-
- Run tests:
-
-    uv run pytest tests

hafnia-0.1.24.dist-info/RECORD

@@ -1,28 +0,0 @@
-cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cli/__main__.py,sha256=h1tOAK15us_dkoMd6Yd4SesVPisojTxOXiYxpLZnatw,1736
-cli/config.py,sha256=Js_dCn39l7hLhA3ovHorOyVqj-LCLzUg_figSy4jNjs,5279
-cli/consts.py,sha256=nd9fPTypqCJYJoOp9QifNgj7-c91J-T0dTv83c33g50,892
-cli/data_cmds.py,sha256=FYZPaeTs6WQnTf9b4a56w3SXQ1VAOMF5s8eUDHeNtd4,1538
-cli/experiment_cmds.py,sha256=d9M8008Z0Y49KIITy73qd5ViPehgMymDVAgsvZhwye0,3099
-cli/profile_cmds.py,sha256=Rg-5wLHSWlZhNPUZBO7LdyJS-Y-SgI6qKLoAac2gSdk,2534
-cli/runc_cmds.py,sha256=fNgPNURXmO4nfLyuorcpqWEeNNuTVsRBHxcIA9FCPts,2197
-hafnia/__init__.py,sha256=Zphq-cQoX95Z11zm4lkrU-YiAJxddR7IBfwDkxeHoDE,108
-hafnia/http.py,sha256=rID6Krn9wRGXwsJYvpffsFlt5cwxFgkcihYppqtdT-8,2974
-hafnia/log.py,sha256=ii--Q6IThsWOluRp_Br9WGhwBtKChU80BXk5pK_NU5A,819
-hafnia/torch_helpers.py,sha256=P_Jl4IwqUebKVCOXNe6iTorJZA3S-3d92HV274UHIko,7456
-hafnia/utils.py,sha256=WWWXZPolzncQmSb4onArT1lJyISV0D22eEejrHWePoc,2425
-hafnia/data/__init__.py,sha256=Pntmo_1fst8OhyrHB60jQ8mhJJ4hL38tdjLvt0YXEJo,73
-hafnia/data/factory.py,sha256=61oGQsm1naG_6Nd_UY2teOki8Oiu2F-gT-nGocmqAcg,2992
-hafnia/experiment/__init__.py,sha256=OEFE6HqhO5zcTCLZcPcPVjIg7wMFFnvZ1uOtAVhRz7M,85
-hafnia/experiment/hafnia_logger.py,sha256=bkSfioEaBXthUEGBjMbQaSefU30b_Gu809VcmWy_wyg,6566
-hafnia/platform/__init__.py,sha256=Oz1abs40hEKspLg6mVIokdtsp1tZJF9Pndv8uSMOgtQ,522
-hafnia/platform/api.py,sha256=aJvlQGjzqm-D3WYb2xTEYX60YoJoWN_kyYdlkvqt_MI,382
-hafnia/platform/builder.py,sha256=VqcbOPxC7HqGAqFMb6ewThBZYLEV5RBgQrVuMd2dbLY,6622
-hafnia/platform/download.py,sha256=AWnlSYj9FD7GvZ_-9Sw5jrcxi3RyBSSUVph8U9T9ZbQ,4711
-hafnia/platform/executor.py,sha256=HA8IF2ZTZ6ZiRWNjVdIdWKiUa5i8Yoz06mIUBDwzVDk,3759
-hafnia/platform/experiment.py,sha256=951ppXdrp075pW2xGFOM0oiGYGE1I53tP9azQjjIUe8,2305
-hafnia-0.1.24.dist-info/METADATA,sha256=85f5fVCG57auPMtzbF6T0o05smjq20XvZP-6jqNqEYg,8124
-hafnia-0.1.24.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-hafnia-0.1.24.dist-info/entry_points.txt,sha256=FCJVIQ8GP2VE9I3eeGVF5eLxVDNW_01pOJCpG_CGnMM,45
-hafnia-0.1.24.dist-info/licenses/LICENSE,sha256=DqQ3NOAy7Efwppv0IAVXEm3Za2SI_1OuDfG20ab8eQw,1078
-hafnia-0.1.24.dist-info/RECORD,,