hafnia 0.1.24__py3-none-any.whl → 0.1.26__py3-none-any.whl

cli/__main__.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 import click
 
-from cli import consts, data_cmds, experiment_cmds, profile_cmds, runc_cmds
+from cli import consts, data_cmds, experiment_cmds, profile_cmds, recipe_cmds, runc_cmds
 from cli.config import Config, ConfigSchema
 
 
@@ -10,6 +10,7 @@ from cli.config import Config, ConfigSchema
 def main(ctx: click.Context) -> None:
     """Hafnia CLI."""
     ctx.obj = Config()
+    ctx.max_content_width = 120
 
 
 @main.command("configure")
@@ -17,9 +18,7 @@ def main(ctx: click.Context) -> None:
 def configure(cfg: Config) -> None:
     """Configure Hafnia CLI settings."""
 
-    from hafnia.platform.api import get_organization_id
-
-    profile_name = click.prompt("Profile Name", type=str, default="default")
+    profile_name = click.prompt("Profile Name", type=str, default=consts.DEFAULT_PROFILE_NAME)
     profile_name = profile_name.strip()
     try:
         cfg.add_profile(profile_name, ConfigSchema(), set_active=True)
@@ -32,12 +31,8 @@ def configure(cfg: Config) -> None:
     except ValueError as e:
         click.echo(f"Error: {str(e)}", err=True)
         return
-    platform_url = click.prompt("Hafnia Platform URL", type=str, default="https://api.mdi.milestonesys.com")
+    platform_url = click.prompt("Hafnia Platform URL", type=str, default=consts.DEFAULT_API_URL)
     cfg.platform_url = platform_url.strip()
-    try:
-        cfg.organization_id = get_organization_id(cfg.get_platform_endpoint("organizations"), cfg.api_key)
-    except Exception:
-        raise click.ClickException(consts.ERROR_ORG_ID)
     cfg.save_config()
     profile_cmds.profile_show(cfg)
 
@@ -54,6 +49,7 @@ main.add_command(profile_cmds.profile)
 main.add_command(data_cmds.data)
 main.add_command(runc_cmds.runc)
 main.add_command(experiment_cmds.experiment)
+main.add_command(recipe_cmds.recipe)
 
 if __name__ == "__main__":
-    main()
+    main(max_content_width=120)

cli/config.py

@@ -6,14 +6,21 @@ from typing import Dict, List, Optional
 from pydantic import BaseModel, field_validator
 
 import cli.consts as consts
-from hafnia.log import logger
+from hafnia.log import user_logger
+
+PLATFORM_API_MAPPING = {
+    "recipes": "/api/v1/recipes",
+    "experiments": "/api/v1/experiments",
+    "experiment_environments": "/api/v1/experiment-environments",
+    "experiment_runs": "/api/v1/experiment-runs",
+    "runs": "/api/v1/experiments-runs",
+    "datasets": "/api/v1/datasets",
+}
 
 
 class ConfigSchema(BaseModel):
-    organization_id: str = ""
     platform_url: str = ""
     api_key: Optional[str] = None
-    api_mapping: Optional[Dict[str, str]] = None
 
     @field_validator("api_key")
     def validate_api_key(cls, value: str) -> str:
@@ -61,14 +68,6 @@ class Config:
     def api_key(self, value: str) -> None:
         self.config.api_key = value
 
-    @property
-    def organization_id(self) -> str:
-        return self.config.organization_id
-
-    @organization_id.setter
-    def organization_id(self, value: str) -> None:
-        self.config.organization_id = value
-
     @property
     def platform_url(self) -> str:
         return self.config.platform_url
@@ -77,7 +76,6 @@ class Config:
     def platform_url(self, value: str) -> None:
         base_url = value.rstrip("/")
         self.config.platform_url = base_url
-        self.config.api_mapping = self.get_api_mapping(base_url)
 
     def __init__(self, config_path: Optional[Path] = None) -> None:
         self.config_path = self.resolve_config_path(config_path)
@@ -96,27 +94,22 @@ class Config:
 
     def add_profile(self, profile_name: str, profile: ConfigSchema, set_active: bool = False) -> None:
         profile_name = profile_name.strip()
+        if profile_name in self.config_data.profiles:
+            user_logger.warning(
+                f"Profile with name '{profile_name}' already exists, it will be overwritten by the new one."
+            )
+
         self.config_data.profiles[profile_name] = profile
         if set_active:
             self.config_data.active_profile = profile_name
         self.save_config()
 
-    def get_api_mapping(self, base_url: str) -> Dict:
-        return {
-            "organizations": f"{base_url}/api/v1/organizations",
-            "recipes": f"{base_url}/api/v1/recipes",
-            "experiments": f"{base_url}/api/v1/experiments",
-            "experiment_environments": f"{base_url}/api/v1/experiment-environments",
-            "experiment_runs": f"{base_url}/api/v1/experiment-runs",
-            "runs": f"{base_url}/api/v1/experiments-runs",
-            "datasets": f"{base_url}/api/v1/datasets",
-        }
-
     def get_platform_endpoint(self, method: str) -> str:
         """Get specific API endpoint"""
-        if not self.config.api_mapping or method not in self.config.api_mapping:
-            raise ValueError(f"{method} is not supported.")
-        return self.config.api_mapping[method]
+        if method not in PLATFORM_API_MAPPING:
+            raise ValueError(f"'{method}' is not supported.")
+        endpoint = self.config.platform_url + PLATFORM_API_MAPPING[method]
+        return endpoint
 
     def load_config(self) -> ConfigFileSchema:
         """Load configuration from file."""
@@ -127,7 +120,7 @@ class Config:
                 data = json.load(f)
             return ConfigFileSchema(**data)
         except json.JSONDecodeError:
-            logger.error("Error decoding JSON file.")
+            user_logger.error("Error decoding JSON file.")
             raise ValueError("Failed to parse configuration file")
 
     def save_config(self) -> None:

cli/consts.py

@@ -1,13 +1,16 @@
+DEFAULT_API_URL = "https://api.mdi.milestonesys.com"
+DEFAULT_PROFILE_NAME = "default"
+
 ERROR_CONFIGURE: str = "Please configure the CLI with `hafnia configure`"
 ERROR_PROFILE_NOT_EXIST: str = "No active profile configured. Please configure the CLI with `hafnia configure`"
 ERROR_PROFILE_REMOVE_ACTIVE: str = "Cannot remove active profile. Please switch to another profile first."
 ERROR_API_KEY_NOT_SET: str = "API key not set. Please configure the CLI with `hafnia configure`."
-ERROR_ORG_ID: str = "Failed to fetch organization ID. Verify platform URL and API key."
 ERROR_CREATE_PROFILE: str = "Failed to create profile. Profile name must be unique and not empty."
 
 ERROR_GET_RESOURCE: str = "Failed to get the data from platform. Verify url or api key."
 
 ERROR_EXPERIMENT_DIR: str = "Source directory does not exist"
+ERROR_RECIPE_FILE_FORMAT: str = "Recipe filename must be a '.zip' file"
 
 PROFILE_SWITCHED_SUCCESS: str = "Switched to profile:"
 PROFILE_REMOVED_SUCCESS: str = "Removed profile:"

cli/data_cmds.py

@@ -1,3 +1,4 @@
+from pathlib import Path
 from typing import Optional
 
 import click
@@ -35,20 +36,18 @@ def data_get(cfg: Config, url: str, destination: click.Path) -> None:
 @click.argument("destination", default=None, required=False)
 @click.option("--force", is_flag=True, default=False, help="Force download")
 @click.pass_obj
-def data_download(cfg: Config, dataset_name: str, destination: Optional[click.Path], force: bool) -> None:
+def data_download(cfg: Config, dataset_name: str, destination: Optional[click.Path], force: bool) -> Path:
     """Download dataset from Hafnia platform"""
 
     from hafnia.data.factory import download_or_get_dataset_path
 
     try:
-        endpoint_dataset = cfg.get_platform_endpoint("datasets")
-        api_key = cfg.api_key
-        download_or_get_dataset_path(
+        path_dataset = download_or_get_dataset_path(
             dataset_name=dataset_name,
-            endpoint=endpoint_dataset,
-            api_key=api_key,
+            cfg=cfg,
             output_dir=destination,
             force_redownload=force,
         )
     except Exception:
         raise click.ClickException(consts.ERROR_GET_RESOURCE)
+    return path_dataset

cli/experiment_cmds.py

@@ -13,32 +13,6 @@ def experiment() -> None:
     pass
 
 
-@experiment.command(name="create_recipe")
-@click.option("--source_folder", default=".", type=Path, help="Path to the source folder", show_default=True)
-@click.option(
-    "--recipe_filename",
-    default="recipe.zip",
-    type=Path,
-    help="Recipe filename. Should have a '.zip' suffix",
-    show_default=True,
-)
-def create_recipe(source_folder: str, recipe_filename: str) -> None:
-    """Build recipe from local path as image with prefix - localhost"""
-
-    from hafnia.platform.builder import validate_recipe
-    from hafnia.utils import archive_dir
-
-    path_output_zip = Path(recipe_filename)
-
-    if path_output_zip.suffix != ".zip":
-        raise click.ClickException("Recipe filename must be a '.zip' file")
-
-    path_source = Path(source_folder)
-
-    path_output_zip = archive_dir(path_source, path_output_zip)
-    validate_recipe(path_output_zip)
-
-
 @experiment.command(name="create")
 @click.argument("name")
 @click.argument("source_dir", type=Path)
@@ -48,25 +22,18 @@ def create_recipe(source_folder: str, recipe_filename: str) -> None:
 @click.pass_obj
 def create(cfg: Config, name: str, source_dir: Path, exec_cmd: str, dataset_name: str, env_name: str) -> None:
     """Create a new experiment run"""
-    from hafnia.platform import (
-        create_experiment,
-        create_recipe,
-        get_dataset_id,
-        get_exp_environment_id,
-    )
+    from hafnia.platform import create_experiment, create_recipe, get_dataset_id, get_exp_environment_id
 
     if not source_dir.exists():
         raise click.ClickException(consts.ERROR_EXPERIMENT_DIR)
 
     try:
         dataset_id = get_dataset_id(dataset_name, cfg.get_platform_endpoint("datasets"), cfg.api_key)
-    except (IndexError, KeyError):
-        raise click.ClickException(f"Dataset '{dataset_name}' not found.")
     except Exception:
         raise click.ClickException(f"Error retrieving dataset '{dataset_name}'.")
 
     try:
-        recipe_id = create_recipe(source_dir, cfg.get_platform_endpoint("recipes"), cfg.api_key, cfg.organization_id)
+        recipe_id = create_recipe(source_dir, cfg.get_platform_endpoint("recipes"), cfg.api_key)
     except Exception:
         raise click.ClickException(f"Failed to create recipe from '{source_dir}'")
 
@@ -77,14 +44,7 @@ def create(cfg: Config, name: str, source_dir: Path, exec_cmd: str, dataset_name
 
     try:
         experiment_id = create_experiment(
-            name,
-            dataset_id,
-            recipe_id,
-            exec_cmd,
-            env_id,
-            cfg.get_platform_endpoint("experiments"),
-            cfg.api_key,
-            cfg.organization_id,
+            name, dataset_id, recipe_id, exec_cmd, env_id, cfg.get_platform_endpoint("experiments"), cfg.api_key
         )
     except Exception:
         raise click.ClickException(f"Failed to create experiment '{name}'")

cli/profile_cmds.py

@@ -3,7 +3,7 @@ from rich.console import Console
 from rich.table import Table
 
 import cli.consts as consts
-from cli.config import Config
+from cli.config import Config, ConfigSchema
 
 
 @click.group()
@@ -43,6 +43,21 @@ def profile_use(cfg: Config, profile_name: str) -> None:
     click.echo(f"{consts.PROFILE_SWITCHED_SUCCESS} {profile_name}")
 
 
+@profile.command("create")
+@click.argument("api-key", required=True)
+@click.option("--name", help="Specify profile name", default=consts.DEFAULT_PROFILE_NAME, show_default=True)
+@click.option("--api-url", help="API URL", default=consts.DEFAULT_API_URL, show_default=True)
+@click.option(
+    "--activate/--no-activate", help="Activate the created profile after creation", default=True, show_default=True
+)
+@click.pass_obj
+def profile_create(cfg: Config, name: str, api_url: str, api_key: str, activate: bool) -> None:
+    """Create a new profile."""
+    cfg_profile = ConfigSchema(platform_url=api_url, api_key=api_key)
+
+    cfg.add_profile(profile_name=name, profile=cfg_profile, set_active=activate)
+
+
 @profile.command("rm")
 @click.argument("profile_name", required=True)
 @click.pass_obj
@@ -80,7 +95,6 @@ def profile_show(cfg: Config) -> None:
     table.add_column("Value")
 
     table.add_row("API Key", masked_key)
-    table.add_row("Organization", cfg.organization_id)
     table.add_row("Platform URL", cfg.platform_url)
     table.add_row("Config File", cfg.config_path.as_posix())
     console.print(table)

cli/recipe_cmds.py

@@ -0,0 +1,45 @@
+from pathlib import Path
+
+import click
+
+import cli.consts as consts
+
+
+@click.group(name="recipe")
+def recipe() -> None:
+    """Hafnia Recipe management commands"""
+    pass
+
+
+@recipe.command(name="create")
+@click.argument("source")
+@click.option(
+    "--output", type=click.Path(writable=True), default="./recipe.zip", show_default=True, help="Output recipe path."
+)
+def create(source: str, output: str) -> None:
+    """Create HRF from local path"""
+
+    from hafnia.utils import archive_dir
+
+    path_output_zip = Path(output)
+    if path_output_zip.suffix != ".zip":
+        raise click.ClickException(consts.ERROR_RECIPE_FILE_FORMAT)
+
+    path_source = Path(source)
+    path_output_zip = archive_dir(path_source, path_output_zip)
+
+
+@recipe.command(name="view")
+@click.option("--path", type=str, default="./recipe.zip", show_default=True, help="Path of recipe.zip.")
+@click.option("--depth-limit", type=int, default=3, help="Limit the depth of the tree view.", show_default=True)
+def view(path: str, depth_limit: int) -> None:
+    """View the content of a recipe zip file."""
+    from hafnia.utils import show_recipe_content
+
+    path_recipe = Path(path)
+    if not path_recipe.exists():
+        raise click.ClickException(
+            f"Recipe file '{path_recipe}' does not exist. Please provide a valid path. "
+            f"To create a recipe, use the 'hafnia recipe create' command."
+        )
+    show_recipe_content(path_recipe, depth_limit=depth_limit)

cli/runc_cmds.py

@@ -1,10 +1,14 @@
-from hashlib import sha256
+import json
+import subprocess
+import zipfile
 from pathlib import Path
 from tempfile import TemporaryDirectory
+from typing import Optional
 
 import click
 
 from cli.config import Config
+from hafnia.log import sys_logger, user_logger
 
 
 @click.group(name="runc")
@@ -13,56 +17,128 @@ def runc():
     pass
 
 
-@runc.command(name="launch")
-@click.argument("task", required=True)
-def launch(task: str) -> None:
+@runc.command(name="launch-local")
+@click.argument("exec_cmd", type=str)
+@click.option(
+    "--dataset",
+    type=str,
+    help="Hafnia dataset name e.g. mnist, midwest-vehicle-detection or a path to a local dataset",
+    required=True,
+)
+@click.option(
+    "--image_name",
+    type=Optional[str],
+    default=None,
+    help=(
+        "Docker image name to use for the launch. "
+        "By default, it will use image name from '.state.json' "
+        "file generated by the 'hafnia runc build-local' command"
+    ),
+)
+@click.pass_obj
+def launch_local(cfg: Config, exec_cmd: str, dataset: str, image_name: str) -> None:
     """Launch a job within the image."""
-    from hafnia.platform.executor import handle_launch
+    from hafnia.data.factory import download_or_get_dataset_path
+
+    is_local_dataset = "/" in dataset
+    if is_local_dataset:
+        click.echo(f"Using local dataset: {dataset}")
+        path_dataset = Path(dataset)
+        if not path_dataset.exists():
+            raise click.ClickException(f"Dataset path does not exist: {path_dataset}")
+    else:
+        click.echo(f"Using Hafnia dataset: {dataset}")
+        path_dataset = download_or_get_dataset_path(dataset_name=dataset, cfg=cfg, force_redownload=False)
+
+    if image_name is None:
+        # Load image name from state.json
+        path_state_file = Path("state.json")
+        if not path_state_file.exists():
+            raise click.ClickException("State file does not exist. Please build the image first.")
+        state_dict = json.loads(path_state_file.read_text())
+        if "image_tag" not in state_dict:
+            raise click.ClickException("'image_tag' not found in state file. Please build the image first.")
+        image_name = state_dict["image_tag"]
 
-    handle_launch(task)
+    docker_cmds = [
+        "docker",
+        "run",
+        "--rm",
+        "-v",
+        f"{path_dataset.absolute()}:/opt/ml/input/data/training",
+        "-e",
+        "HAFNIA_CLOUD=true",
+        "-e",
+        "PYTHONPATH=src",
+        "--runtime",
+        "nvidia",
+        image_name,
+    ] + exec_cmd.split(" ")
+
+    # Use the "hafnia runc launch" cmd when we have moved to the new folder structure and
+    # direct commands.
+    # Replace '+ exec_cmd.split(" ")' with '["hafnia", "runc", "launch"] + exec_cmd.split(" ")'
+
+    click.echo(f"Running command: \n\t{' '.join(docker_cmds)}")
+    subprocess.run(docker_cmds, check=True)
 
 
 @runc.command(name="build")
 @click.argument("recipe_url")
-@click.argument("state_file", default="state.json")
-@click.argument("ecr_repository", default="localhost")
-@click.argument("image_name", default="recipe")
+@click.option("--state_file", "--st", type=str, default="state.json")
+@click.option("--repo", type=str, default="localhost", help="Docker repository")
 @click.pass_obj
-def build(cfg: Config, recipe_url: str, state_file: str, ecr_repository: str, image_name: str) -> None:
+def build(cfg: Config, recipe_url: str, state_file: str, repo: str) -> None:
     """Build docker image with a given recipe."""
     from hafnia.platform.builder import build_image, prepare_recipe
 
     with TemporaryDirectory() as temp_dir:
-        image_info = prepare_recipe(recipe_url, Path(temp_dir), cfg.api_key)
-        image_info["name"] = image_name
-        build_image(image_info, ecr_repository, state_file)
+        metadata = prepare_recipe(recipe_url, Path(temp_dir), cfg.api_key)
+        build_image(metadata, repo, state_file=state_file)
 
 
 @runc.command(name="build-local")
 @click.argument("recipe")
-@click.argument("state_file", default="state.json")
-@click.argument("image_name", default="recipe")
-def build_local(recipe: str, state_file: str, image_name: str) -> None:
+@click.option("--state_file", "--st", type=str, default="state.json")
+@click.option("--repo", type=str, default="localhost", help="Docker repository")
+def build_local(recipe: Path, state_file: str, repo: str) -> None:
     """Build recipe from local path as image with prefix - localhost"""
+    import shutil
+    import uuid
+
+    import seedir
+
+    from hafnia.platform.builder import build_image
+    from hafnia.utils import filter_recipe_files
+
+    recipe = Path(recipe)
+
+    with TemporaryDirectory() as d:
+        tmp_dir = Path(d)
+        recipe_dir = tmp_dir / "recipe"
+        recipe_dir.mkdir(parents=True, exist_ok=True)
+
+        if recipe.suffix == ".zip":
+            user_logger.info("Extracting recipe for processing.")
+            with zipfile.ZipFile(recipe.as_posix(), "r") as zip_ref:
+                zip_ref.extractall(recipe_dir)
+        elif recipe.is_dir():
+            for rf in filter_recipe_files(recipe):
+                src_path = (recipe / rf).absolute()
+                target_path = recipe_dir / rf
+                target_path.parent.mkdir(parents=True, exist_ok=True)
+                shutil.copyfile(src_path, target_path)
+
+        user_logger.info(
+            seedir.seedir(recipe_dir, sort=True, first="folders", style="emoji", printout=False, depthlimit=2)
+        )
+
+        metadata = {
+            "dockerfile": (recipe_dir / "Dockerfile").as_posix(),
+            "docker_context": recipe_dir.as_posix(),
+            "digest": uuid.uuid4().hex[:8],
+        }
 
-    from hafnia.platform.builder import build_image, validate_recipe
-    from hafnia.utils import archive_dir
-
-    recipe_zip = Path(recipe)
-    recipe_created = False
-    if not recipe_zip.suffix == ".zip" and recipe_zip.is_dir():
-        recipe_zip = archive_dir(recipe_zip)
-        recipe_created = True
-
-    validate_recipe(recipe_zip)
-    click.echo("Recipe successfully validated")
-    image_info = {
-        "name": image_name,
-        "dockerfile": f"{recipe_zip.parent}/Dockerfile",
-        "docker_context": f"{recipe_zip.parent}",
-        "hash": sha256(recipe_zip.read_bytes()).hexdigest()[:8],
-    }
-    click.echo("Start building image")
-    build_image(image_info, "localhost", state_file=state_file)
-    if recipe_created:
-        recipe_zip.unlink()
+        user_logger.info("Start building image.")
+        sys_logger.debug(metadata)
+        build_image(metadata, repo, state_file=state_file)

hafnia/data/factory.py

@@ -7,7 +7,7 @@ from datasets import Dataset, DatasetDict, load_from_disk
 
 from cli.config import Config
 from hafnia import utils
-from hafnia.log import logger
+from hafnia.log import user_logger
 from hafnia.platform import download_resource, get_dataset_id
 
 
@@ -15,29 +15,33 @@ def load_local(dataset_path: Path) -> Union[Dataset, DatasetDict]:
     """Load a Hugging Face dataset from a local directory path."""
     if not dataset_path.exists():
         raise ValueError(f"Can not load dataset, directory does not exist -- {dataset_path}")
-    logger.info(f"Loading data from {dataset_path.as_posix()}")
+    user_logger.info(f"Loading data from {dataset_path.as_posix()}")
     return load_from_disk(dataset_path.as_posix())
 
 
 def download_or_get_dataset_path(
     dataset_name: str,
-    endpoint: str,
-    api_key: str,
+    cfg: Optional[Config] = None,
     output_dir: Optional[str] = None,
     force_redownload: bool = False,
 ) -> Path:
     """Download or get the path of the dataset."""
+
+    cfg = cfg or Config()
+    endpoint_dataset = cfg.get_platform_endpoint("datasets")
+    api_key = cfg.api_key
+
     output_dir = output_dir or str(utils.PATH_DATASET)
     dataset_path_base = Path(output_dir).absolute() / dataset_name
     dataset_path_base.mkdir(exist_ok=True, parents=True)
     dataset_path_sample = dataset_path_base / "sample"
 
     if dataset_path_sample.exists() and not force_redownload:
-        logger.info("Dataset found locally. Set 'force=True' or add `--force` flag with cli to re-download")
+        user_logger.info("Dataset found locally. Set 'force=True' or add `--force` flag with cli to re-download")
         return dataset_path_sample
 
-    dataset_id = get_dataset_id(dataset_name, endpoint, api_key)
-    dataset_access_info_url = f"{endpoint}/{dataset_id}/temporary-credentials"
+    dataset_id = get_dataset_id(dataset_name, endpoint_dataset, api_key)
+    dataset_access_info_url = f"{endpoint_dataset}/{dataset_id}/temporary-credentials"
 
     if force_redownload and dataset_path_sample.exists():
         # Remove old files to avoid old files conflicting with new files
@@ -48,23 +52,6 @@ def download_or_get_dataset_path(
     raise RuntimeError("Failed to download dataset")
 
 
-def load_from_platform(
-    dataset_name: str,
-    endpoint: str,
-    api_key: str,
-    output_dir: Optional[str] = None,
-    force_redownload: bool = False,
-) -> Union[Dataset, DatasetDict]:
-    path_dataset = download_or_get_dataset_path(
-        dataset_name=dataset_name,
-        endpoint=endpoint,
-        api_key=api_key,
-        output_dir=output_dir,
-        force_redownload=force_redownload,
-    )
-    return load_local(path_dataset)
-
-
 def load_dataset(dataset_name: str, force_redownload: bool = False) -> Union[Dataset, DatasetDict]:
     """Load a dataset either from a local path or from the Hafnia platform."""
 
@@ -72,15 +59,9 @@ def load_dataset(dataset_name: str, force_redownload: bool = False) -> Union[Dat
         path_dataset = Path(os.getenv("MDI_DATASET_DIR", "/opt/ml/input/data/training"))
         return load_local(path_dataset)
 
-    cfg = Config()
-    endpoint_dataset = cfg.get_platform_endpoint("datasets")
-    api_key = cfg.api_key
-    dataset = load_from_platform(
+    path_dataset = download_or_get_dataset_path(
         dataset_name=dataset_name,
-        endpoint=endpoint_dataset,
-        api_key=api_key,
-        output_dir=None,
         force_redownload=force_redownload,
     )
-
+    dataset = load_local(path_dataset)
     return dataset

hafnia/experiment/hafnia_logger.py

@@ -13,7 +13,7 @@ from datasets import DatasetDict
 from pydantic import BaseModel, field_validator
 
 from hafnia.data.factory import load_dataset
-from hafnia.log import logger
+from hafnia.log import sys_logger, user_logger
 from hafnia.utils import is_remote_job, now_as_str
 
 
@@ -49,7 +49,7 @@ class Entity(BaseModel):
         try:
             return float(v)
         except (ValueError, TypeError) as e:
-            logger.warning(f"Invalid value '{v}' provided, defaulting to -1.0: {e}")
+            user_logger.warning(f"Invalid value '{v}' provided, defaulting to -1.0: {e}")
             return -1.0
 
     @field_validator("ent_type", mode="before")
@@ -159,11 +159,15 @@ class HafniaLogger:
     def log_hparams(self, params: Dict, fname: str = "hparams.json"):
         file_path = self._path_artifacts() / fname
         try:
-            with open(file_path, "w") as f:
-                json.dump(params, f, indent=2)
-            logger.info(f"Saved parameters to {file_path}")
+            if file_path.exists():  # New params are appended to existing params
+                existing_params = json.loads(file_path.read_text())
+            else:
+                existing_params = {}
+            existing_params.update(params)
+            file_path.write_text(json.dumps(existing_params, indent=2))
+            user_logger.info(f"Saved parameters to {file_path}")
         except Exception as e:
-            logger.error(f"Failed to save parameters to {file_path}: {e}")
+            user_logger.error(f"Failed to save parameters to {file_path}: {e}")
 
     def log_environment(self):
         environment_info = {
@@ -197,4 +201,4 @@ class HafniaLogger:
                 next_table = pa.concat_tables([prev, log_batch])
                 pq.write_table(next_table, self.log_file)
         except Exception as e:
-            logger.error(f"Failed to flush logs: {e}")
+            sys_logger.error(f"Failed to flush logs: {e}")