ahorn-loader 0.4.1__py3-none-any.whl

ahorn_loader/__init__.py

@@ -0,0 +1,3 @@
+"""Library and CLI for loading and managing AHORN datasets."""
+
+from .api import *

ahorn_loader/api.py

@@ -0,0 +1,261 @@
+"""Module to interact with the Ahorn dataset API."""
+
+import contextlib
+import gzip
+import json
+import logging
+from collections.abc import Generator, Iterator
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import TypedDict
+from urllib.parse import ParseResult, urlparse
+
+import httpx
+from httpx_retries import Retry, RetryTransport
+
+from .utils import get_cache_dir
+
+__all__ = [
+    "download_dataset",
+    "load_dataset_data",
+    "load_datasets_data",
+    "read_dataset",
+]
+
+DATASET_API_URL = "https://ahorn.rwth-aachen.de/api/datasets.json"
+
+logger = logging.getLogger(__name__)
+
+
+class AttachmentDict(TypedDict):
+    url: str
+    size: int
+
+
+class DatasetDict(TypedDict):
+    slug: str
+    title: str
+    tags: list[str]
+    attachments: dict[str, AttachmentDict]
+
+
+class DatasetsDataDict(TypedDict):
+    datasets: dict[str, DatasetDict]
+    time: str
+
+
+def load_datasets_data(*, cache_lifetime: int | None = None) -> dict[str, DatasetDict]:
+    """Load dataset data from the Ahorn API.
+
+    Parameters
+    ----------
+    cache_lifetime : int, optional
+        How long to reuse cached data in seconds. If not provided, the cache will not
+        be used.
+
+    Returns
+    -------
+    dict[str, Any]
+        Dictionary containing dataset information, where the keys are dataset slugs
+        and the values are dictionaries with dataset details such as title, tags, and
+        attachments.
+    """
+    datasets_data_cache = get_cache_dir() / "datasets.json"
+    if datasets_data_cache.exists() and cache_lifetime is not None:
+        cache_mtime = datetime.fromtimestamp(
+            datasets_data_cache.stat().st_mtime, tz=UTC
+        )
+        age_seconds = (datetime.now(tz=UTC) - cache_mtime).total_seconds()
+        if age_seconds < cache_lifetime:
+            logger.info(
+                "Using cached datasets list (age=%.1fs, lifetime=%ss)",
+                age_seconds,
+                cache_lifetime,
+            )
+            with datasets_data_cache.open("r", encoding="utf-8") as cache_file:
+                cache: DatasetsDataDict = json.load(cache_file)
+                return cache["datasets"]
+
+    logger.info("Fetching datasets list from %s", DATASET_API_URL)
+    response = httpx.get(DATASET_API_URL, timeout=10)
+    response.raise_for_status()
+
+    datasets_data_cache.parent.mkdir(parents=True, exist_ok=True)
+    with datasets_data_cache.open("w", encoding="utf-8") as cache_file:
+        cache_file.write(response.text)
+
+    response_json: DatasetsDataDict = response.json()
+    return response_json["datasets"]
+
+
+def load_dataset_data(slug: str, *, cache_lifetime: int | None = None) -> DatasetDict:
+    """Load data for a specific dataset by its slug.
+
+    Parameters
+    ----------
+    slug : str
+        The slug of the dataset to load.
+    cache_lifetime : int, optional
+        How long to reuse cached data in seconds. If not provided, the cache will not
+        be used.
+
+    Returns
+    -------
+    DatasetDict
+        Dictionary containing the dataset details.
+
+    Raises
+    ------
+    KeyError
+        If the dataset with the given `slug` does not exist.
+    """
+    datasets = load_datasets_data(cache_lifetime=cache_lifetime)
+
+    if slug not in datasets:
+        raise KeyError(f"Dataset with slug '{slug}' does not exist in AHORN.")
+
+    return datasets[slug]
+
+
+def get_dataset_url(slug: str, *, cache_lifetime: int | None = None) -> str:
+    """Get the download URL for a specific dataset by its slug.
+
+    Parameters
+    ----------
+    slug : str
+        The slug of the dataset.
+    cache_lifetime : int, optional
+        How long to reuse cached data in seconds. If not provided, the cache will not
+        be used.
+
+    Returns
+    -------
+    str
+        The download URL of the dataset.
+
+    Raises
+    ------
+    KeyError
+        If the dataset with the given `slug` does not exist.
+    RuntimeError
+        If the dataset does not contain the required attachment information.
+    """
+    data = load_dataset_data(slug, cache_lifetime=cache_lifetime)
+    if "dataset" not in data["attachments"]:
+        raise RuntimeError(
+            f"Dataset '{slug}' does not contain required 'attachments/dataset' keys."
+        )
+    return data["attachments"]["dataset"]["url"]
+
+
+def download_dataset(
+    slug: str, folder: Path | str, *, cache_lifetime: int | None = None
+) -> Path:
+    """Download a dataset by its slug to the specified folder.
+
+    This function implements an exponential backoff strategy when encountering HTTP 429
+    (Too Many Requests) responses. If available, it respects the 'Retry-After' header to
+    determine the wait time before retrying.
+
+    Parameters
+    ----------
+    slug : str
+        The slug of the dataset to download.
+    folder : Path | str
+        The folder where the dataset should be saved.
+    cache_lifetime : int, optional
+        How long to reuse cached data in seconds. If not provided, the cache will not
+        be used.
+
+    Returns
+    -------
+    Path
+        The path to the downloaded dataset file.
+
+    Raises
+    ------
+    KeyError
+        If the dataset with the given `slug` does not exist.
+    HTTPError
+        If the dataset file could not be downloaded due to some error.
+    RuntimeError
+        If the dataset file could not be downloaded due to some error.
+    """
+    if isinstance(folder, str):
+        folder = Path(folder)
+
+    logger.info("Preparing download for dataset '%s' into %s", slug, folder)
+    download_url = get_dataset_url(slug, cache_lifetime=cache_lifetime)
+
+    url: ParseResult = urlparse(download_url)
+    folder.mkdir(parents=True, exist_ok=True)
+    filepath = folder / url.path.split("/")[-1]
+
+    # Use RetryTransport to automatically handle rate limiting (429) with exponential
+    # backoff. This also automatically respects 'Retry-After' headers if provided.
+    retry = Retry(
+        total=5,
+        backoff_factor=2.0,
+    )
+    retry_transport = RetryTransport(retry=retry)
+
+    with (
+        httpx.Client(transport=retry_transport, timeout=10) as client,
+        client.stream("GET", download_url) as response,
+    ):
+        response.raise_for_status()
+
+        with filepath.open("wb") as f:
+            for chunk in response.iter_bytes(chunk_size=8192):
+                f.write(chunk)
+
+    logger.info("Downloaded dataset '%s' to %s", slug, filepath)
+
+    return filepath
+
+
+@contextlib.contextmanager
+def read_dataset(slug: str) -> Generator[Iterator[str], None, None]:
+    """Download and yield a context-managed file object for the dataset lines by slug.
+
+    The dataset file will be stored in your system cache and can be deleted according
+    to your system's cache policy. To ensure that costly re-downloads do not occur, use
+    the `download_dataset` function to store the dataset file at a more permanent
+    location.
+
+    Parameters
+    ----------
+    slug : str
+        The slug of the dataset to download.
+
+    Returns
+    -------
+    Context manager yielding an open file object (iterator over lines).
+
+    Raises
+    ------
+    KeyError
+        If the dataset with the given `slug` does not exist.
+    RuntimeError
+        If the dataset file could not be downloaded due to other errors.
+
+    Examples
+    --------
+    >>> import ahorn_loader
+    >>> with ahorn_loader.read_dataset("contact-high-school") as dataset:
+    >>>     for line in dataset:
+    >>>         ...
+    """
+    download_url = get_dataset_url(slug)
+    filepath = get_cache_dir() / download_url.split("/")[-1]
+
+    # Download the dataset if it is not already cached
+    if not filepath.exists():
+        filepath = download_dataset(slug, get_cache_dir())
+
+    if filepath.suffix == ".gz":
+        with gzip.open(filepath, mode="rt", encoding="utf-8") as f:
+            yield f
+    else:
+        with filepath.open("r", encoding="utf-8") as f:
+            yield f

ahorn_loader/cli.py

@@ -0,0 +1,82 @@
+"""Entry point for the ``ahorn-loader`` command-line application."""
+
+from pathlib import Path
+from typing import Annotated
+
+import typer
+from rich import print as rich_print
+from rich.table import Table
+
+from .api import download_dataset, load_datasets_data
+from .validator import Validator
+
+app = typer.Typer()
+
+
+@app.command()
+def ls() -> None:
+    """List available datasets in AHORN."""
+    try:
+        datasets = load_datasets_data(cache_lifetime=3600)
+        if "error" in datasets:
+            typer.echo(f"Error: {datasets['error']}")
+            raise typer.Exit(code=1)
+    except Exception as e:
+        print(f"Failed to load datasets: {e}")
+        raise typer.Exit(code=1) from e
+
+    table = Table(title="Available Datasets")
+    table.add_column("Slug", style="cyan")
+    table.add_column("Title", style="magenta")
+    table.add_column("Tags", style="green")
+
+    for slug, details in datasets.items():
+        table.add_row(slug, details["title"], ", ".join(details["tags"]))
+    rich_print(table)
+
+
+@app.command()
+def download(
+    name: Annotated[str, typer.Argument(help="The name of the dataset to download.")],
+    folder: Annotated[
+        Path, typer.Argument(help="Folder where the dataset should be saved.")
+    ] = Path(),
+) -> None:
+    """Download the specified dataset from AHORN.
+
+    Parameters
+    ----------
+    name : str
+        The name of the dataset to download.
+    folder : Path
+        The folder where the dataset should be saved. Defaults to the current directory.
+    """
+    try:
+        download_dataset(name, folder, cache_lifetime=3600)
+        typer.echo(f"Downloaded dataset to {folder.absolute()}")
+    except Exception as e:
+        typer.echo(f"Failed to download dataset: {e}")
+        raise typer.Exit(code=1) from e
+
+
+@app.command()
+def validate(
+    path: Annotated[
+        Path, typer.Argument(help="The path to the dataset file to validate.")
+    ],
+) -> None:
+    """Validate whether a given file is a valid AHORN dataset.
+
+    Parameters
+    ----------
+    path : Path
+        The path to the dataset file to validate.
+    """
+    validator = Validator()
+    if not validator.validate(path):
+        typer.echo("Validation failed.")
+        raise typer.Exit(code=1)
+
+
+if __name__ == "__main__":
+    app()

ahorn_loader/utils/__init__.py

@@ -0,0 +1,3 @@
+"""Utility functions used internally in `ahorn-loader`."""
+
+from .cache import *

ahorn_loader/utils/cache.py

@@ -0,0 +1,29 @@
+"""Module with cache-related utility functions."""
+
+import os
+import sys
+from pathlib import Path
+
+__all__ = ["get_cache_dir"]
+
+
+def get_cache_dir() -> Path:
+    """Return an appropriate cache location for the current platform.
+
+    Returns
+    -------
+    pathlib.Path
+        Platform-dependent cache directory.
+    """
+    match sys.platform:
+        case "win32":
+            base = os.getenv("LOCALAPPDATA") or Path("~\\AppData\\Local").expanduser()
+            return Path(base) / "ahorn-loader" / "Cache"
+        case "darwin":
+            return Path.home() / "Library" / "Caches" / "ahorn-loader"
+        case _:
+            # Linux and other Unix
+            xdg = os.getenv("XDG_CACHE_HOME")
+            if xdg:
+                return Path(xdg) / "ahorn-loader"
+            return Path.home() / ".cache" / "ahorn-loader"

ahorn_loader/validator/__init__.py

@@ -0,0 +1,3 @@
+"""Module containing the validator for AHORN datasets."""
+
+from .validator import *

ahorn_loader/validator/rules.py

@@ -0,0 +1,119 @@
+"""Module with validation rules for a AHORN dataset."""
+
+import json
+import logging
+from abc import ABC, abstractmethod
+from pathlib import Path
+
+__all__ = [
+    "DatasetRule",
+    "FileNameRule",
+    "NetworkLevelMetadataRule",
+    "PreFlightRule",
+]
+
+
+class PreFlightRule(ABC):
+    """Base class for validation rules that run before the dataset file is loaded."""
+
+    def __init__(self) -> None:
+        self.logger = logging.getLogger(self.__class__.__name__)
+
+    @abstractmethod
+    def validate(self, file_path: Path) -> bool:
+        """
+        Validate the dataset before loading it.
+
+        Parameters
+        ----------
+        file_path : pathlib.Path
+            The path of the file to validate.
+
+        Returns
+        -------
+        bool
+            True if the dataset is valid, False otherwise.
+        """
+
+
+class DatasetRule(ABC):
+    """Base class for validation rules that validates the dataset content."""
+
+    def __init__(self) -> None:
+        self.logger = logging.getLogger(self.__class__.__name__)
+
+    @abstractmethod
+    def validate(self, content: list[str]) -> bool:
+        """
+        Validate the dataset content.
+
+        Parameters
+        ----------
+        content : list[str]
+            The content of the dataset file to validate.
+
+        Returns
+        -------
+        bool
+            True if the dataset content is valid, False otherwise.
+        """
+
+
+class FileNameRule(PreFlightRule):
+    """Rule to validate file names."""
+
+    def validate(self, file_path: Path) -> bool:
+        """
+        Validate the file name against a specific pattern.
+
+        Parameters
+        ----------
+        file_path : pathlib.Path
+            The path of the file to validate.
+
+        Returns
+        -------
+        bool
+            True if the file name is valid, False otherwise.
+        """
+        if not (file_path.suffix == ".txt" or file_path.name.endswith(".txt.gz")):
+            self.logger.error("Dataset must be a .txt or .txt.gz file.")
+            return False
+
+        # TODO: Check that the file can be read as plain text or as gzipped text.
+
+        self.logger.debug("File name %s is valid.", file_path.name)
+        return True
+
+
+class NetworkLevelMetadataRule(DatasetRule):
+    """Rule to validate network-level metadata."""
+
+    def validate(self, content: list[str]) -> bool:
+        """
+        Validate the network-level metadata.
+
+        Parameters
+        ----------
+        content : list[str]
+            The content of the dataset file to validate.
+
+        Returns
+        -------
+        bool
+            True if the metadata is valid, False otherwise.
+        """
+        try:
+            metadata = json.loads(content[0])
+        except json.JSONDecodeError:
+            self.logger.error("First line of the dataset must be valid JSON metadata.")
+            return False
+        self.logger.debug(
+            "Parsed network-level metadata successfully.", extra={"metadata": metadata}
+        )
+
+        if "_format_version" not in metadata:
+            self.logger.error("Network-level metadata must contain '_format_version'.")
+            return False
+
+        return True

ahorn_loader/validator/validator.py

@@ -0,0 +1,54 @@
+"""Module containing the validator for AHORN datasets."""
+
+import gzip
+from pathlib import Path
+
+from .rules import DatasetRule, FileNameRule, NetworkLevelMetadataRule, PreFlightRule
+
+__all__ = ["Validator"]
+
+
+class Validator:
+    """Validator class to manage validation rules."""
+
+    pre_flight_rules: list[PreFlightRule]
+    dataset_rules: list[DatasetRule]
+
+    def __init__(self) -> None:
+        self.pre_flight_rules = [
+            FileNameRule(),
+        ]
+
+        self.dataset_rules = [
+            NetworkLevelMetadataRule(),
+        ]
+
+    def validate(self, dataset_path: Path | str) -> bool:
+        """Run all validation rules.
+
+        Parameters
+        ----------
+        dataset_path : Path | str
+            The path to the dataset file to validate.
+
+        Returns
+        -------
+        bool
+            True if all validation rules pass, False otherwise.
+        """
+        if isinstance(dataset_path, str):
+            dataset_path = Path(dataset_path)
+
+        if not all(
+            rule.validate(file_path=dataset_path) for rule in self.pre_flight_rules
+        ):
+            return False
+
+        if dataset_path.suffix == ".gz":
+            with gzip.open(dataset_path, "rt") as f:
+                content = f.readlines()
+        else:
+            with dataset_path.open() as f:
+                content = f.readlines()
+
+        return all(rule.validate(content=content) for rule in self.dataset_rules)

ahorn_loader-0.4.1.dist-info/METADATA

@@ -0,0 +1,85 @@
+Metadata-Version: 2.4
+Name: ahorn-loader
+Version: 0.4.1
+Summary: Library and command-line application to interact with datasets in the Aachen Higher-Order Repository of Networks.
+Author: Florian Frantzen
+Author-email: Florian Frantzen <frantzen@netsci.rwth-aachen.de>
+License-Expression: MIT
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Scientific/Engineering
+Requires-Dist: httpx>=0.27.0
+Requires-Dist: httpx-retries>=0.4.5
+Requires-Dist: typer>=0.16.0
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+
+# `ahorn-loader`
+
+Library and command-line application to interact with datasets in [AHORN](https://ahorn.rwth-aachen.de/).
+
+<div align="center">
+
+[![Python](https://img.shields.io/badge/python-3.12+-blue)](https://www.python.org/)
+[![license](https://badgen.net/github/license/netsci-rwth/ahorn-loader)](https://github.com/netsci-rwth/ahorn-loader/blob/main/LICENSE)
+
+</div>
+
+## Usage
+
+`ahorn-loader` is both a command-line application and a Python package to interact with the AHORN repository for higher-order datasets.
+
+### Command-Line Usage
+
+To install and use `ahorn-loader` from the command line, you can run the following command:
+
+```bash
+uvx ahorn-loader [command] [args]
+```
+
+Commands include:
+
+- `ls`: List available datasets in AHORN.
+- `download`: Download a dataset from AHORN.
+- `validate`: Validate a specific dataset file (e.g., before adding it to AHORN).
+
+To get a full help of available commands and options, run `ahorn-loader --help`.
+
+### Python Package Usage
+
+To use `ahorn-loader` as a Python package, you can install it via `pip` (or some other package manager of your choice):
+
+```bash
+pip install ahorn-loader
+```
+
+Then, you can use it in your Python scripts:
+
+```python
+import ahorn_loader
+
+# Download a dataset:
+ahorn_loader.download_dataset("dataset_name", "target_path")
+
+# Download and read a dataset:
+# The dataset will be stored in your system's cache. For a more permanent storage
+# location, use `ahorn_loader.download_dataset` instead.
+with ahorn_loader.read_dataset("dataset_name") as dataset:
+    for line in dataset:
+        ...
+```
+
+## Funding
+
+<img align="right" width="200" src="https://raw.githubusercontent.com/netsci-rwth/ahorn/main/public/images/erc_logo.png">
+
+Funded by the European Union (ERC, HIGH-HOPeS, 101039827).
+Views and opinions expressed are however those of the author(s) only and do not necessarily reflect those of the European Union or the European Research Council Executive Agency.
+Neither the European Union nor the granting authority can be held responsible for them.

ahorn_loader-0.4.1.dist-info/RECORD

@@ -0,0 +1,13 @@
+ahorn_loader/__init__.py,sha256=kEDhV6uY5P7i2ceFDSPi7CCR9GekRszv7EzvYx4RDEw,83
+ahorn_loader/api.py,sha256=emtkn7dEvFZaNspRLdnO94St0-84qRrQIY4cF3BEP04,7831
+ahorn_loader/cli.py,sha256=4fFIQVhE-Zzvq47JMghKoMFAzhZXJ8lXRdtyAjvYzBY,2272
+ahorn_loader/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ahorn_loader/utils/__init__.py,sha256=kIYHc-9ExuESHM2TIXlh9-YF7r7hFiRfjAKYTQG4gGg,81
+ahorn_loader/utils/cache.py,sha256=rRsn5z6LM1aFLufZGM4uppHVP553iR8cP3JTNxZiEKY,832
+ahorn_loader/validator/__init__.py,sha256=tyGbqMMzzkGPI3pEb9uBAJoNMGUds_WdU_5575vGBM8,84
+ahorn_loader/validator/rules.py,sha256=djiWi4_Y-UlC2XhwPGrZywyr56AoPfAcNpOnNMZ6w8I,3155
+ahorn_loader/validator/validator.py,sha256=qfooTPfjZ2ieqraJ3CqdqADfDFlODHm-OU_LRPK0gmM,1437
+ahorn_loader-0.4.1.dist-info/WHEEL,sha256=XV0cjMrO7zXhVAIyyc8aFf1VjZ33Fen4IiJk5zFlC3g,80
+ahorn_loader-0.4.1.dist-info/entry_points.txt,sha256=oyQAA_k5r0sAD_lBKgQLPhpxqk0-UTagDJlsU97AJ4s,55
+ahorn_loader-0.4.1.dist-info/METADATA,sha256=k_zmzKGL7SpBu-m8sR1QMmwhQKf8-ubAEYPaVdreCAM,2973
+ahorn_loader-0.4.1.dist-info/RECORD,,

ahorn_loader-0.4.1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: uv 0.9.26
+Root-Is-Purelib: true
+Tag: py3-none-any

ahorn_loader-0.4.1.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+ahorn-loader = ahorn_loader.cli:app
+