ahorn-loader 0.1.0__tar.gz

ahorn_loader-0.1.0/PKG-INFO

@@ -0,0 +1,53 @@
+Metadata-Version: 2.3
+Name: ahorn-loader
+Version: 0.1.0
+Summary: Add your description here
+Author: Florian Frantzen
+Author-email: Florian Frantzen <florian.frantzen@cs.rwth-aachen.de>
+Requires-Dist: requests>=2.32.4
+Requires-Dist: typer>=0.16.0
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+
+# `ahorn-loader`
+
+Library and command-line application to interact with datasets in [AHORN](https://ahorn.rwth-aachen.de/).
+
+## Usage
+
+`ahorn-loader` is both a command-line application and a Python package to interact with the AHORN repository for higher-order datasets.
+
+### Command-Line Usage
+
+To install and use `ahorn-loader` from the command line, you can run the following command:
+
+```bash
+uvx ahorn-loader [command] [args]
+```
+
+Commands include:
+- `ls`: List available datasets in AHORN.
+- `download`: Download a dataset from AHORN.
+- `validate`: Validate a specific dataset file (e.g., before adding it to AHORN).
+
+To get a full help of available commands and options, run `ahorn-loader --help`.
+
+### Python Package Usage
+
+To use `ahorn-loader` as a Python package, you can install it via `pip` (or some other package manager of your choice):
+
+```bash
+pip install ahorn-loader
+```
+
+Then, you can use it in your Python scripts:
+
+```python
+import ahorn_loader
+
+# download a dataset
+ahorn_loader.download('dataset_name', 'target_path')
+
+# validate a specific dataset (e.g., before adding it to AHORN)
+ahorn_loader.validate('path_to_dataset_file')
+```

ahorn_loader-0.1.0/README.md

@@ -0,0 +1,42 @@
+# `ahorn-loader`
+
+Library and command-line application to interact with datasets in [AHORN](https://ahorn.rwth-aachen.de/).
+
+## Usage
+
+`ahorn-loader` is both a command-line application and a Python package to interact with the AHORN repository for higher-order datasets.
+
+### Command-Line Usage
+
+To install and use `ahorn-loader` from the command line, you can run the following command:
+
+```bash
+uvx ahorn-loader [command] [args]
+```
+
+Commands include:
+- `ls`: List available datasets in AHORN.
+- `download`: Download a dataset from AHORN.
+- `validate`: Validate a specific dataset file (e.g., before adding it to AHORN).
+
+To get a full help of available commands and options, run `ahorn-loader --help`.
+
+### Python Package Usage
+
+To use `ahorn-loader` as a Python package, you can install it via `pip` (or some other package manager of your choice):
+
+```bash
+pip install ahorn-loader
+```
+
+Then, you can use it in your Python scripts:
+
+```python
+import ahorn_loader
+
+# download a dataset
+ahorn_loader.download('dataset_name', 'target_path')
+
+# validate a specific dataset (e.g., before adding it to AHORN)
+ahorn_loader.validate('path_to_dataset_file')
+```

ahorn_loader-0.1.0/pyproject.toml

@@ -0,0 +1,98 @@
+[project]
+name = "ahorn-loader"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+authors = [
+    { name = "Florian Frantzen", email = "florian.frantzen@cs.rwth-aachen.de" }
+]
+requires-python = ">=3.12"
+dependencies = [
+    "requests>=2.32.4",
+    "typer>=0.16.0",
+]
+
+[project.scripts]
+ahorn-loader = "ahorn_loader.cli:app"
+
+[build-system]
+requires = ["uv_build>=0.8.4,<0.9.0"]
+build-backend = "uv_build"
+
+[dependency-groups]
+dev = [
+    "mypy>=1.17.1",
+    "pytest>=8.4.1",
+    "ruff>=0.12.7",
+    "types-requests>=2.32.4",
+]
+
+[tool.mypy]
+warn_redundant_casts = true
+warn_unreachable = true
+warn_unused_ignores = true
+enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"]
+
+[tool.pytest.ini_options]
+minversion = "7.0"
+addopts = ["-ra", "--showlocals", "--strict-markers", "--strict-config"]
+xfail_strict = true
+filterwarnings = ["error"]
+log_cli_level = "info"
+testpaths = ["tests"]
+
+[tool.ruff.format]
+docstring-code-format = true
+
+[tool.ruff.lint]
+select = [
+    "F", # pyflakes errors
+    "E", # code style
+    "W", # warnings
+    "I", # import order
+    "D", # pydocstyle rules
+    "UP", # pyupgrade rules
+    "YTT", # flake8-2020 rules
+    "S", # bandit rules
+    "BLE", # blind except
+    "B", # bugbear rules
+    "A", # builtin shadowing
+    "COM", # comma rules
+    "C4", # comprehensions
+    "DTZ", # datetime rules
+    "T10", # debugger calls
+    "FA", # future annotations
+    "ISC", # implicit str concatenation
+    "ICN", # import conventions
+    "LOG", # logging rules
+    "G", # logging format rules
+    "PIE", # pie rules
+    "Q", # quote rules
+    "RSE", # raise rules
+    "RET", # return rules
+    "SLOT", # slot rules
+    "SIM", # code simplifications
+    "TID", # tidy imports
+    "TC", # type checking rules
+    "PTH", # use pathlib
+    "PD", # pandas rules
+    "PLC", # pylint conventions
+    "PLE", # pylint errors
+    "FLY", # flynt
+    "NPY", # numpy rules
+    "PERF", # performance rules
+    "FURB", # refurb
+    "RUF", # miscellaneous rules
+]
+ignore = [
+    "E501",    # line too long
+    "COM812",  # trailing commas; conflict with `ruff format`
+    "ISC001",  # implicitly single-line str concat; conflict with `ruff format`
+]
+
+[tool.ruff.lint.per-file-ignores]
+"__init__.py" = ["F403"]
+"tests/**.py" = ["S101"]
+
+[tool.ruff.lint.pydocstyle]
+convention = "numpy"

ahorn_loader-0.1.0/src/ahorn_loader/__init__.py

@@ -0,0 +1,3 @@
+"""Library and CLI for loading and managing AHORN datasets."""
+
+from .api import *

ahorn_loader-0.1.0/src/ahorn_loader/api.py

@@ -0,0 +1,111 @@
+"""Module to interact with the Ahorn dataset API."""
+
+import json
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+import requests
+
+__all__ = ["download_dataset", "load_dataset_data", "load_datasets_data"]
+
+DATASET_API_URL = "https://ahorn.rwth-aachen.de/api/datasets.json"
+CACHE_PATH = Path(__file__).parent.parent.parent / "cache" / "datasets.json"
+
+
+def load_datasets_data(*, cache_lifetime: int | None = None) -> dict[str, Any]:
+    """Load dataset data from the Ahorn API.
+
+    Parameters
+    ----------
+    cache_lifetime : int, optional
+        How long to reuse cached data in seconds. If not provided, the cache will not
+        be used.
+
+    Returns
+    -------
+    dict[str, Any]
+        Dictionary containing dataset information, where the keys are dataset slugs
+        and the values are dictionaries with dataset details such as title, tags, and
+        attachments.
+    """
+    if CACHE_PATH.exists() and cache_lifetime is not None:
+        with CACHE_PATH.open("r", encoding="utf-8") as cache_file:
+            cache = json.load(cache_file)
+        if (
+            cache.get("time")
+            and (
+                datetime.now(tz=UTC) - datetime.fromisoformat(cache["time"])
+            ).total_seconds()
+            < cache_lifetime
+        ):
+            return cache["datasets"]
+
+    response = requests.get(DATASET_API_URL, timeout=10)
+    response.raise_for_status()
+
+    CACHE_PATH.parent.mkdir(parents=True, exist_ok=True)
+    with CACHE_PATH.open("w", encoding="utf-8") as cache_file:
+        cache_file.write(response.text)
+
+    return response.json()["datasets"]
+
+
+def load_dataset_data(
+    slug: str, *, cache_lifetime: int | None = None
+) -> dict[str, Any]:
+    """Load data for a specific dataset by its slug.
+
+    Parameters
+    ----------
+    slug : str
+        The slug of the dataset to load.
+    cache_lifetime : int, optional
+        How long to reuse cached data in seconds. If not provided, the cache will not
+        be used.
+
+    Returns
+    -------
+    dict[str, Any]
+        Dictionary containing the dataset details.
+    """
+    datasets = load_datasets_data(cache_lifetime=cache_lifetime)
+    if "error" in datasets:
+        return {"error": datasets["error"]}
+
+    return datasets.get(slug, {"error": f"Dataset '{slug}' not found."})
+
+
+def download_dataset(
+    slug: str, folder: Path | str, *, cache_lifetime: int | None = None
+) -> None:
+    """Download a dataset by its slug to the specified folder.
+
+    Parameters
+    ----------
+    slug : str
+        The slug of the dataset to download.
+    folder : Path | str
+        The folder where the dataset should be saved.
+    cache_lifetime : int, optional
+        How long to reuse cached data in seconds. If not provided, the cache will not
+        be used.
+    """
+    if isinstance(folder, str):
+        folder = Path(folder)
+
+    data = load_dataset_data(slug, cache_lifetime=cache_lifetime)
+    if "error" in data:
+        raise ValueError(f"Error loading dataset '{slug}': {data['error']}")
+    if "attachments" not in data or "dataset" not in data["attachments"]:
+        raise KeyError(f"Dataset '{slug}' does not contain required 'attachments/dataset' keys.")
+    dataset_attachment = data["attachments"]["dataset"]
+
+    response = requests.get(dataset_attachment["url"], timeout=10, stream=True)
+    response.raise_for_status()
+    folder.mkdir(parents=True, exist_ok=True)
+    filepath = folder / dataset_attachment["name"]
+    with filepath.open("wb") as f:
+        for chunk in response.iter_content(chunk_size=8192):
+            if chunk:
+                f.write(chunk)

ahorn_loader-0.1.0/src/ahorn_loader/cli.py

@@ -0,0 +1,83 @@
+"""Entry point for the ``ahorn-loader`` command-line application."""
+
+from pathlib import Path
+from typing import Annotated
+
+import typer
+from rich import print as rich_print
+from rich.table import Table
+
+from .api import download_dataset, load_datasets_data
+from .validator import Validator
+
+app = typer.Typer()
+
+
+@app.command()
+def ls() -> None:
+    """List available datasets in AHORN."""
+    try:
+        datasets = load_datasets_data(cache_lifetime=3600)
+        if "error" in datasets:
+            typer.echo(f"Error: {datasets['error']}")
+            raise typer.Exit(code=1)
+    except Exception as e:
+        print(f"Failed to load datasets: {e}")
+        raise typer.Exit(code=1) from e
+
+    table = Table(title="Available Datasets")
+    table.add_column("Slug", style="cyan")
+    table.add_column("Title", style="magenta")
+    table.add_column("Tags", style="green")
+
+    for slug, details in datasets.items():
+        table.add_row(slug, details["title"], ", ".join(details["tags"]))
+    rich_print(table)
+
+
+@app.command()
+def download(
+    name: Annotated[str, typer.Argument(help="The name of the dataset to download.")],
+    folder: Annotated[
+        Path, typer.Argument(help="Folder where the dataset should be saved.")
+    ] = Path(),
+) -> None:
+    """Download the specified dataset from AHORN.
+
+    Parameters
+    ----------
+    name : str
+        The name of the dataset to download.
+    folder : Path
+        The folder where the dataset should be saved. Defaults to the current directory.
+    """
+    download_dataset(name, folder, cache_lifetime=3600)
+    typer.echo(f"Downloaded dataset to {folder}")
+
+    try:
+        download_dataset(name, folder, cache_lifetime=3600)
+        typer.echo(f"Downloaded dataset to {folder}")
+    except Exception as e:
+        typer.echo(f"Failed to download dataset: {e}")
+        raise typer.Exit(code=1) from e
+@app.command()
+def validate(
+    path: Annotated[
+        Path, typer.Argument(help="The path to the dataset file to validate.")
+    ],
+) -> None:
+    """Validate whether a given file is a valid AHORN dataset.
+
+    Parameters
+    ----------
+    path : Path
+        The path to the dataset file to validate.
+    """
+    validator = Validator()
+    if not validator.validate(path):
+        typer.echo("Validation failed.")
+        raise typer.Exit(code=1)
+
+
+if __name__ == "__main__":
+    app()

ahorn_loader-0.1.0/src/ahorn_loader/validator/__init__.py

@@ -0,0 +1,3 @@
+"""Module containing the validator for AHORN datasets."""
+
+from .validator import *

ahorn_loader-0.1.0/src/ahorn_loader/validator/rules.py

@@ -0,0 +1,119 @@
+"""Module with validation rules for a AHORN dataset."""
+
+import json
+import logging
+from abc import ABC, abstractmethod
+from pathlib import Path
+
+__all__ = [
+    "DatasetRule",
+    "FileNameRule",
+    "NetworkLevelMetadataRule",
+    "PreFlightRule",
+]
+
+
+class PreFlightRule(ABC):
+    """Base class for validation rules that run before the dataset file is loaded."""
+
+    def __init__(self) -> None:
+        self.logger = logging.getLogger(self.__class__.__name__)
+
+    @abstractmethod
+    def validate(self, file_path: Path) -> bool:
+        """
+        Validate the dataset before loading it.
+
+        Parameters
+        ----------
+        file_path : pathlib.Path
+            The path of the file to validate.
+
+        Returns
+        -------
+        bool
+            True if the dataset is valid, False otherwise.
+        """
+
+
+class DatasetRule(ABC):
+    """Base class for validation rules that validates the dataset content."""
+
+    def __init__(self) -> None:
+        self.logger = logging.getLogger(self.__class__.__name__)
+
+    @abstractmethod
+    def validate(self, content: list[str]) -> bool:
+        """
+        Validate the dataset content.
+
+        Parameters
+        ----------
+        content : list[str]
+            The content of the dataset file to validate.
+
+        Returns
+        -------
+        bool
+            True if the dataset content is valid, False otherwise.
+        """
+
+
+class FileNameRule(PreFlightRule):
+    """Rule to validate file names."""
+
+    def validate(self, file_path: Path) -> bool:
+        """
+        Validate the file name against a specific pattern.
+
+        Parameters
+        ----------
+        file_path : pathlib.Path
+            The path of the file to validate.
+
+        Returns
+        -------
+        bool
+            True if the file name is valid, False otherwise.
+        """
+        if not (file_path.suffix == ".txt" or file_path.name.endswith(".txt.gz")):
+            self.logger.error("Dataset must be a .txt or .txt.gz file.")
+            return False
+
+        # TODO: Check that the file can be read as plain text or as gzipped text.
+
+        self.logger.debug("File name %s is valid.", file_path.name)
+        return True
+
+
+class NetworkLevelMetadataRule(DatasetRule):
+    """Rule to validate network-level metadata."""
+
+    def validate(self, content: list[str]) -> bool:
+        """
+        Validate the network-level metadata.
+
+        Parameters
+        ----------
+        content : list[str]
+            The content of the dataset file to validate.
+
+        Returns
+        -------
+        bool
+            True if the metadata is valid, False otherwise.
+        """
+        try:
+            metadata = json.loads(content[0])
+        except json.JSONDecodeError:
+            self.logger.error("First line of the dataset must be valid JSON metadata.")
+            return False
+        self.logger.debug(
+            "Parsed network-level metadata successfully.", extra={"metadata": metadata}
+        )
+
+        if "_format_version" not in metadata:
+            self.logger.error("Network-level metadata must contain '_format_version'.")
+            return False
+
+        return True

ahorn_loader-0.1.0/src/ahorn_loader/validator/validator.py

@@ -0,0 +1,54 @@
+"""Module containing the validator for AHORN datasets."""
+
+import gzip
+from pathlib import Path
+
+from .rules import DatasetRule, FileNameRule, NetworkLevelMetadataRule, PreFlightRule
+
+__all__ = ["Validator"]
+
+
+class Validator:
+    """Validator class to manage validation rules."""
+
+    pre_flight_rules: list[PreFlightRule]
+    dataset_rules: list[DatasetRule]
+
+    def __init__(self) -> None:
+        self.pre_flight_rules = [
+            FileNameRule(),
+        ]
+
+        self.dataset_rules = [
+            NetworkLevelMetadataRule(),
+        ]
+
+    def validate(self, dataset_path: Path | str) -> bool:
+        """Run all validation rules.
+
+        Parameters
+        ----------
+        dataset_path : Path | str
+            The path to the dataset file to validate.
+
+        Returns
+        -------
+        bool
+            True if all validation rules pass, False otherwise.
+        """
+        if isinstance(dataset_path, str):
+            dataset_path = Path(dataset_path)
+
+        if not all(
+            rule.validate(file_path=dataset_path) for rule in self.pre_flight_rules
+        ):
+            return False
+
+        if dataset_path.suffix == ".gz":
+            with gzip.open(dataset_path, "rt") as f:
+                content = f.readlines()
+        else:
+            with dataset_path.open() as f:
+                content = f.readlines()
+
+        return all(rule.validate(content=content) for rule in self.dataset_rules)