ilostat 0.1.0__py3-none-any.whl

ilostat/__init__.py

@@ -0,0 +1,21 @@
+from importlib.metadata import version
+
+from .cache import clean_cache, get_cache_dir
+from .data import get_df
+from .dictionary import get_dataset_dic, get_dic
+from .metadata import get_metadata, get_toc
+from .sdmx import get_sdmx
+
+__version__ = version("ilostat")
+
+__all__ = [
+    "__version__",
+    "get_dataset_dic",
+    "get_df",
+    "get_dic",
+    "get_metadata",
+    "get_toc",
+    "clean_cache",
+    "get_cache_dir", 
+    "get_sdmx",
+]

ilostat/cache.py

@@ -0,0 +1,183 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from platformdirs import user_cache_dir
+
+
+def _format_last_update(
+    last_update: str,
+) -> str:
+    """
+    Convert an ILOSTAT last.update value to a cache timestamp.
+
+    Example
+    -------
+    '20/09/2026 07:34:56' -> '20260920T073456'
+    """
+    date_part, time_part = last_update.split(" ")
+
+    day, month, year = date_part.split("/")
+    hour, minute, second = time_part.split(":")
+
+    return f"{year}{month}{day}T{hour}{minute}{second}"
+
+
+def _versioned_cache_filename(
+    dataset_id: str,
+    last_update: str,
+    suffix: str,
+) -> str:
+    """
+    Build a versioned cache filename.
+
+    Examples
+    --------
+    Dataset:
+    UNE_TUNE_SEX_AGE_NB_A-20260930T070804.parquet
+
+    Dictionary:
+    UNE_TUNE_SEX_AGE_NB_A-20260930T070804_dic.json
+    """
+    timestamp = _format_last_update(
+        last_update
+    )
+
+    return (
+        f"{dataset_id}-"
+        f"{timestamp}"
+        f"{suffix}"
+    )
+
+
+def _cache_filename(
+    dataset_id: str,
+    last_update: str,
+) -> str:
+    """
+    Build the filename for a cached ILOSTAT dataset.
+    """
+    return _versioned_cache_filename(
+        dataset_id=dataset_id,
+        last_update=last_update,
+        suffix=".parquet",
+    )
+
+
+def _remove_old_cache_files(
+    cache_dir: Path,
+    dataset_id: str,
+    keep_file: Path,
+    suffix: str = ".parquet",
+) -> None:
+    """
+    Remove obsolete cached versions of an ILOSTAT resource.
+
+    Parameters
+    ----------
+    cache_dir : pathlib.Path
+        Segment cache directory.
+
+    dataset_id : str
+        ILOSTAT dataset identifier.
+
+    keep_file : pathlib.Path
+        Current cached file that must be preserved.
+
+    suffix : str, default ".parquet"
+        Resource filename suffix.
+    """
+    pattern = (
+        f"{dataset_id}-"
+        f"*{suffix}"
+    )
+
+    for file in cache_dir.glob(
+        pattern
+    ):
+        if file != keep_file:
+            file.unlink()
+
+
+def get_cache_dir(
+    cache_dir: str | Path | None = None,
+) -> Path:
+    """
+    Return the ILOSTAT cache directory.
+
+    Parameters
+    ----------
+    cache_dir : str, pathlib.Path or None
+        Custom cache directory. If None, use the operating
+        system's standard user cache directory.
+
+    Returns
+    -------
+    pathlib.Path
+        Cache directory.
+    """
+    if cache_dir is None:
+        path = Path(
+            user_cache_dir("ilostat")
+        )
+    else:
+        path = Path(
+            cache_dir
+        ).expanduser()
+
+    path.mkdir(
+        parents=True,
+        exist_ok=True,
+    )
+
+    return path
+
+
+def clean_cache(
+    cache_dir: str | Path | None = None,
+) -> None:
+    """
+    Remove all files from the ILOSTAT cache.
+
+    Parameters
+    ----------
+    cache_dir : str, pathlib.Path or None
+        ILOSTAT cache directory to clean.
+
+        If None, clean the default ILOSTAT user cache
+        directory.
+
+        If provided, clean the specified cache directory.
+
+    Notes
+    -----
+    The cache directory itself is preserved.
+    """
+    path = get_cache_dir(
+        cache_dir
+    )
+
+    for file in path.rglob("*"):
+        if file.is_file():
+            file.unlink()
+
+    # Remove empty subdirectories, deepest first.
+    directories = sorted(
+        (
+            directory
+            for directory in path.rglob("*")
+            if directory.is_dir()
+        ),
+        key=lambda directory: len(
+            directory.parts
+        ),
+        reverse=True,
+    )
+
+    for directory in directories:
+        try:
+            directory.rmdir()
+        except OSError:
+            # Keep non-empty or temporarily locked
+            # directories.
+            pass

ilostat/data.py

@@ -0,0 +1,260 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import pandas as pd
+import requests
+
+from .cache import (
+    _cache_filename,
+    _remove_old_cache_files,
+    get_cache_dir,
+)
+from .dictionary import get_dataset_dic
+from .filters import apply_filters
+from .labels import add_labels, labels_only
+from .metadata import get_last_update
+
+BASE_URL = "https://rplumber.ilo.org/files"
+
+
+def _get_df_single(
+    dataset_id: str,
+    segment: str = "indicator",
+    type: str = "code",
+    lang: str = "en",
+    filters: dict | None = None,
+    best_source: str = "yes",
+    fixed: bool = True,
+    cache_dir: str | Path | None = None,
+    refresh: bool = False,
+) -> pd.DataFrame:
+    """
+    Retrieve an ILOSTAT dataset as a pandas DataFrame.
+
+    Parameters
+    ----------
+    dataset_id : str
+        ILOSTAT dataset identifier.
+
+    segment : str, default "indicator"
+        ILOSTAT bulk-data segment.
+
+    type : {"code", "label", "both"}, default "code"
+        Output representation for coded dimensions.
+
+        "code" returns codes only.
+        "both" returns codes and labels.
+        "label" returns labelled dimensions.
+
+    lang : {"en", "fr", "es"}, default "en"
+        Language used for labels.
+
+    filters : dict or None, default None
+        Filters applied to the dataset.
+
+    best_source : {"yes", "no", "all"}, default "yes"
+        Select best-source observations.
+
+    fixed : bool, default True
+        If True, treat filter values as literal strings.
+
+    cache_dir : str, pathlib.Path or None, default None
+        Directory used to cache downloaded files.
+
+    refresh : bool, default False
+        If True, refresh the table of contents before checking
+        the cached dataset.
+
+    Returns
+    -------
+    pandas.DataFrame
+        Requested ILOSTAT dataset.
+    """
+    type = type.lower()
+
+    if type not in {"code", "label", "both"}:
+        raise ValueError(
+            "Invalid type value. "
+            "Choose from 'code', 'label', or 'both'."
+        )
+
+    cache_path = get_cache_dir(cache_dir)
+
+    last_update = get_last_update(
+        dataset_id=dataset_id,
+        segment=segment,
+        cache_dir=cache_dir,
+        refresh=refresh,
+    )
+
+    if last_update is None:
+        raise ValueError(
+            f"Dataset '{dataset_id}' was not found "
+            f"in the '{segment}' table of contents."
+        )
+
+    filename = _cache_filename(
+        dataset_id=dataset_id,
+        last_update=last_update,
+    )
+
+    segment_dir = cache_path / segment
+    segment_dir.mkdir(
+        parents=True,
+        exist_ok=True,
+    )
+
+    dataset_file = segment_dir / filename
+
+    if not dataset_file.exists():
+        url = (
+            f"{BASE_URL}/{segment}/"
+            f"{dataset_id}.parquet"
+        )
+
+        response = requests.get(
+            url,
+            timeout=60,
+        )
+        response.raise_for_status()
+
+        dataset_file.write_bytes(
+            response.content
+        )
+
+        _remove_old_cache_files(
+            cache_dir=segment_dir,
+            dataset_id=dataset_id,
+            keep_file=dataset_file,
+        )
+
+    df = pd.read_parquet(dataset_file)
+
+    best_source = best_source.lower()
+
+    if best_source not in {"yes", "no", "all"}:
+        raise ValueError(
+            "Invalid best_source value. "
+            "Choose from 'yes', 'no', or 'all'."
+        )
+
+    if "best_source" in df.columns:
+        if best_source == "yes":
+            df = df.loc[
+                df["best_source"] == 1
+            ]
+        elif best_source == "no":
+            df = df.loc[
+                df["best_source"] == 0
+            ]
+
+    df = apply_filters(
+        df,
+        filters=filters,
+        fixed=fixed,
+    )
+
+    if type in {"label", "both"}:
+        dic = get_dataset_dic(
+            dataset_id=dataset_id,
+            segment=segment,
+            cache_dir=cache_dir,
+            refresh=refresh,
+        )
+
+        df = add_labels(
+            df,
+            dic,
+            lang=lang,
+        )
+
+        if type == "label":
+            df = labels_only(df)
+
+    if best_source == "yes":
+        df = df.drop(
+            columns=["best_source"],
+            errors="ignore",
+        )
+
+    return df
+
+def get_df(
+    dataset_id: str | list[str] | tuple[str, ...],
+    segment: str = "indicator",
+    type: str = "code",
+    lang: str = "en",
+    filters: dict | None = None,
+    best_source: str = "yes",
+    fixed: bool = True,
+    cache_dir: str | Path | None = None,
+    refresh: bool = False,
+) -> pd.DataFrame:
+    """
+    Retrieve one or more ILOSTAT datasets.
+
+    Parameters
+    ----------
+    dataset_id : str, list of str, or tuple of str
+        One or more ILOSTAT dataset identifiers.
+
+    segment : str, default "indicator"
+        ILOSTAT bulk-data segment.
+
+    type : {"code", "label", "both"}, default "code"
+        Output representation for coded dimensions.
+
+    lang : {"en", "fr", "es"}, default "en"
+        Language used for labels.
+
+    filters : dict or None, default None
+        Filters applied to each dataset.
+
+    best_source : {"yes", "no", "all"}, default "yes"
+        Select best-source observations.
+
+    fixed : bool, default True
+        If True, treat filter values as literal strings.
+
+    cache_dir : str, pathlib.Path or None, default None
+        Directory used to cache downloaded files.
+
+    refresh : bool, default False
+        If True, refresh metadata before checking cached data.
+
+    Returns
+    -------
+    pandas.DataFrame
+        Requested ILOSTAT data.
+    """
+    if isinstance(dataset_id, str):
+        dataset_ids = [dataset_id]
+    else:
+        dataset_ids = list(dataset_id)
+
+    if not dataset_ids:
+        raise ValueError(
+            "dataset_id must contain at least one dataset identifier."
+        )
+
+    frames = [
+        _get_df_single(
+            dataset_id=current_id,
+            segment=segment,
+            type=type,
+            lang=lang,
+            filters=filters,
+            best_source=best_source,
+            fixed=fixed,
+            cache_dir=cache_dir,
+            refresh=refresh,
+        )
+        for current_id in dataset_ids
+    ]
+
+    return pd.concat(
+        frames,
+        ignore_index=True,
+        sort=False,
+    )

ilostat/dictionary.py

@@ -0,0 +1,193 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+import pandas as pd
+import requests
+
+from .cache import (
+    _remove_old_cache_files,
+    _versioned_cache_filename,
+    get_cache_dir,
+)
+from .metadata import get_last_update
+
+BASE_URL = "https://rplumber.ilo.org/files/dic"
+DATA_BASE_URL = "https://rplumber.ilo.org/files"
+
+VALID_LANGUAGES = {"en", "fr", "es"}
+
+_REFRESHED_DICS: set[tuple[str, str, str]] = set()
+
+
+def get_dic(
+    dic: str,
+    lang: str = "en",
+    cache_dir: str | Path | None = None,
+    refresh: bool = False,
+) -> pd.DataFrame:
+    """
+    Retrieve an ILOSTAT dictionary.
+
+    Parameters
+    ----------
+    dic : str
+        Dictionary name, for example "ref_area", "sex",
+        or "classif_type".
+
+    lang : {"en", "fr", "es"}, default "en"
+        Dictionary language.
+
+    cache_dir : str, pathlib.Path or None, default None
+        Directory used to cache downloaded dictionaries.
+
+    refresh : bool, default False
+        If True, force a new download of the dictionary.
+
+    Returns
+    -------
+    pandas.DataFrame
+        Requested ILOSTAT dictionary.
+    """
+    lang = lang.lower()
+
+    if lang not in VALID_LANGUAGES:
+        raise ValueError(
+            f"Unsupported language '{lang}'. "
+            f"Choose from {sorted(VALID_LANGUAGES)}."
+        )
+
+    cache_path = get_cache_dir(cache_dir)
+
+    dic_dir = cache_path / "dic"
+    dic_dir.mkdir(
+        parents=True,
+        exist_ok=True,
+    )
+
+    dic_file = dic_dir / f"{dic}_{lang}.parquet"
+
+    cache_key = (
+        str(cache_path.resolve()),
+        dic,
+        lang,
+    )
+
+    should_download = (
+        refresh
+        or cache_key not in _REFRESHED_DICS
+        or not dic_file.exists()
+    )
+
+    if should_download:
+        url = (
+            f"{BASE_URL}/"
+            f"{dic}_{lang}.parquet"
+        )
+
+        response = requests.get(
+            url,
+            timeout=60,
+        )
+        response.raise_for_status()
+
+        dic_file.write_bytes(
+            response.content
+        )
+
+        _REFRESHED_DICS.add(cache_key)
+
+    return pd.read_parquet(dic_file)
+
+
+def get_dataset_dic(
+    dataset_id: str,
+    segment: str = "indicator",
+    cache_dir: str | Path | None = None,
+    refresh: bool = False,
+) -> dict[str, Any]:
+    """
+    Retrieve the dataset-specific ILOSTAT dictionary.
+
+    The cached dictionary is versioned using the dataset
+    last.update value from the table of contents.
+
+    Parameters
+    ----------
+    dataset_id : str
+        ILOSTAT dataset identifier.
+
+    segment : str, default "indicator"
+        ILOSTAT data segment.
+
+    cache_dir : str, pathlib.Path or None, default None
+        Directory used to cache downloaded files.
+
+    refresh : bool, default False
+        If True, refresh the table of contents before checking
+        the cached dictionary.
+
+    Returns
+    -------
+    dict
+        Dataset-specific dictionary.
+    """
+    cache_path = get_cache_dir(cache_dir)
+
+    last_update = get_last_update(
+        dataset_id=dataset_id,
+        segment=segment,
+        cache_dir=cache_dir,
+        refresh=refresh,
+    )
+
+    if last_update is None:
+        raise ValueError(
+            f"Dataset '{dataset_id}' was not found "
+            f"in the '{segment}' table of contents."
+        )
+
+    segment_dir = cache_path / segment
+    segment_dir.mkdir(
+        parents=True,
+        exist_ok=True,
+    )
+
+    filename = _versioned_cache_filename(
+        dataset_id=dataset_id,
+        last_update=last_update,
+        suffix="_dic.json",
+    )
+
+    dic_file = segment_dir / filename
+
+    if not dic_file.exists():
+        url = (
+            f"{DATA_BASE_URL}/{segment}/"
+            f"{dataset_id}_dic.json"
+        )
+
+        response = requests.get(
+            url,
+            timeout=60,
+        )
+        response.raise_for_status()
+
+        dic_file.write_bytes(
+            response.content
+        )
+
+        _remove_old_cache_files(
+            cache_dir=segment_dir,
+            dataset_id=dataset_id,
+            keep_file=dic_file,
+            suffix="_dic.json",
+        )
+
+    with dic_file.open(
+        "r",
+        encoding="utf-8",
+    ) as file:
+        return json.load(file)