scmora-db 0.1.0__py3-none-any.whl

scmora_db/__init__.py

@@ -0,0 +1,51 @@
+"""Search, download, and load SCMORA .h5mu datasets."""
+
+from ._version import __version__
+from .catalog import (
+    DEFAULT_REPO_ID,
+    MatchResult,
+    list_dataset_ids,
+    list_detail_sources,
+    list_detailed_conditions,
+    list_usage_tags,
+    list_values,
+    load_catalog,
+    resolve_matches,
+    search_datasets,
+)
+from .exceptions import AmbiguousDatasetError, ScmoraDbError, TooManyMatchesError
+
+__all__ = [
+    "AmbiguousDatasetError",
+    "DEFAULT_REPO_ID",
+    "MatchResult",
+    "ScmoraDbError",
+    "TooManyMatchesError",
+    "__version__",
+    "download_datasets",
+    "list_dataset_ids",
+    "list_detail_sources",
+    "list_detailed_conditions",
+    "list_usage_tags",
+    "list_values",
+    "load_catalog",
+    "load_datasets",
+    "resolve_matches",
+    "search_datasets",
+]
+
+
+def download_datasets(*args, **kwargs):
+    """Download matching .h5mu files from Hugging Face."""
+
+    from .download import download_datasets as _download_datasets
+
+    return _download_datasets(*args, **kwargs)
+
+
+def load_datasets(*args, **kwargs):
+    """Download and load matching .h5mu files with mudata.read_h5mu."""
+
+    from .io import load_datasets as _load_datasets
+
+    return _load_datasets(*args, **kwargs)

scmora_db/_version.py

@@ -0,0 +1,3 @@
+"""Package version."""
+
+__version__ = "0.1.0"

scmora_db/catalog.py

@@ -0,0 +1,288 @@
+"""Metadata catalog access and filtering."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Iterable, List, Optional, Union
+
+import pandas as pd
+
+from .exceptions import AmbiguousDatasetError, TooManyMatchesError
+
+DEFAULT_REPO_ID = "shiny321/genome-db"
+DEFAULT_REPO_TYPE = "dataset"
+DEFAULT_METADATA_FILENAME = "metadata.csv"
+DEFAULT_MAX_AUTO_MATCHES = 5
+LISTABLE_FIELDS = {
+    "condition": "condition",
+    "dataset-id": "dataset_id",
+    "dataset-ids": "dataset_id",
+    "dataset-uid": "dataset_uid",
+    "dataset-uids": "dataset_uid",
+    "detailed-condition": "detailed_condition",
+    "detailed-conditions": "detailed_condition",
+    "detail-source": "detail_source",
+    "detail-sources": "detail_source",
+    "group": "group",
+    "groups": "group",
+    "gse-id": "gse_id",
+    "gse-ids": "gse_id",
+    "reference": "reference",
+    "references": "reference",
+    "sample-type": "sample_type",
+    "sample-types": "sample_type",
+    "species": "species",
+    "usage-primary": "usage_primary",
+    "usage-primaries": "usage_primary",
+    "usage-tag": "usage_tags",
+    "usage-tags": "usage_tags",
+}
+
+
+@dataclass(frozen=True)
+class MatchResult:
+    """A resolved query and the matching rows."""
+
+    rows: pd.DataFrame
+    matched_ids: List[str]
+
+    @property
+    def count(self) -> int:
+        return len(self.rows)
+
+    @property
+    def is_single(self) -> bool:
+        return self.count == 1
+
+
+def load_catalog(
+    repo_id: str = DEFAULT_REPO_ID,
+    *,
+    revision: Optional[str] = None,
+    token: Optional[Union[str, bool]] = None,
+    cache_dir: Optional[Union[str, Path]] = None,
+    metadata_path: Optional[Union[str, Path]] = None,
+    prefer_remote: bool = False,
+) -> pd.DataFrame:
+    """Load the dataset metadata catalog.
+
+    By default this reads the metadata bundled with the package. Set
+    ``prefer_remote=True`` to download ``metadata.csv`` from Hugging Face first.
+    """
+
+    if metadata_path is not None:
+        path = Path(metadata_path)
+    elif prefer_remote:
+        from huggingface_hub import hf_hub_download
+
+        path = Path(
+            hf_hub_download(
+                repo_id=repo_id,
+                repo_type=DEFAULT_REPO_TYPE,
+                filename=DEFAULT_METADATA_FILENAME,
+                revision=revision,
+                token=token,
+                cache_dir=cache_dir,
+            )
+        )
+    else:
+        path = Path(__file__).with_name(DEFAULT_METADATA_FILENAME)
+
+    return _normalize_catalog(_read_metadata_csv(path))
+
+
+def search_datasets(
+    *,
+    dataset_id: Optional[Union[str, Iterable[str]]] = None,
+    dataset_uid: Optional[Union[str, Iterable[str]]] = None,
+    gse_id: Optional[Union[str, Iterable[str]]] = None,
+    detailed_condition: Optional[Union[str, Iterable[str]]] = None,
+    usage_tag: Optional[Union[str, Iterable[str]]] = None,
+    detail_source: Optional[Union[str, Iterable[str]]] = None,
+    condition: Optional[Union[str, Iterable[str]]] = None,
+    sample_type: Optional[Union[str, Iterable[str]]] = None,
+    species: Optional[Union[str, Iterable[str]]] = None,
+    reference: Optional[Union[str, Iterable[str]]] = None,
+    repo_id: str = DEFAULT_REPO_ID,
+    revision: Optional[str] = None,
+    token: Optional[Union[str, bool]] = None,
+    cache_dir: Optional[Union[str, Path]] = None,
+    metadata_path: Optional[Union[str, Path]] = None,
+    prefer_remote: bool = False,
+    case_sensitive: bool = False,
+) -> pd.DataFrame:
+    """Search datasets by metadata fields.
+
+    ``usage_tag`` matches individual semicolon-separated tags in ``usage_tags``.
+    Other text filters use exact matching by default, case-insensitively.
+    """
+
+    df = load_catalog(
+        repo_id=repo_id,
+        revision=revision,
+        token=token,
+        cache_dir=cache_dir,
+        metadata_path=metadata_path,
+        prefer_remote=prefer_remote,
+    )
+
+    filters = {
+        "dataset_uid": dataset_uid,
+        "dataset_id": dataset_id,
+        "gse_id": gse_id,
+        "detailed_condition": detailed_condition,
+        "detail_source": detail_source,
+        "condition": condition,
+        "sample_type": sample_type,
+        "species": species,
+        "reference": reference,
+    }
+    for column, values in filters.items():
+        if values is not None:
+            df = df[_isin(df[column], values, case_sensitive=case_sensitive)]
+
+    if usage_tag is not None:
+        df = df[_has_usage_tag(df["usage_tags"], usage_tag, case_sensitive=case_sensitive)]
+
+    return df.reset_index(drop=True)
+
+
+def resolve_matches(
+    *,
+    max_auto_matches: int = DEFAULT_MAX_AUTO_MATCHES,
+    require_unique_dataset_id: bool = False,
+    **search_kwargs,
+) -> MatchResult:
+    """Resolve a query for download/load operations."""
+
+    rows = search_datasets(**search_kwargs)
+    matched_ids = rows["dataset_uid"].astype(str).tolist()
+
+    if rows.empty:
+        return MatchResult(rows=rows, matched_ids=matched_ids)
+
+    if require_unique_dataset_id and search_kwargs.get("dataset_id") is not None:
+        if len(rows) > 1 and search_kwargs.get("dataset_uid") is None and search_kwargs.get("gse_id") is None:
+            raise AmbiguousDatasetError(search_kwargs["dataset_id"], matched_ids)
+
+    if len(rows) > max_auto_matches:
+        raise TooManyMatchesError(len(rows), matched_ids, max_auto_matches)
+
+    return MatchResult(rows=rows, matched_ids=matched_ids)
+
+
+def list_dataset_ids(**catalog_kwargs) -> List[str]:
+    """Return sorted dataset IDs."""
+
+    return list_values("dataset-id", **catalog_kwargs)
+
+
+def list_detailed_conditions(**catalog_kwargs) -> List[str]:
+    """Return sorted detailed conditions."""
+
+    return list_values("detailed-condition", **catalog_kwargs)
+
+
+def list_detail_sources(**catalog_kwargs) -> List[str]:
+    """Return sorted detail sources."""
+
+    return list_values("detail-source", **catalog_kwargs)
+
+
+def list_usage_tags(**catalog_kwargs) -> List[str]:
+    """Return sorted individual usage tags."""
+
+    return list_values("usage-tag", **catalog_kwargs)
+
+
+def list_values(field: str, **catalog_kwargs) -> List[str]:
+    """Return sorted unique values for a metadata field.
+
+    ``field`` accepts CLI-style names such as ``usage-tags`` and metadata column
+    names such as ``usage_tags``.
+    """
+
+    normalized = field.strip().replace("_", "-")
+    column = LISTABLE_FIELDS.get(normalized, field.strip())
+    df = load_catalog(**catalog_kwargs)
+
+    if column not in df.columns:
+        choices = ", ".join(sorted(LISTABLE_FIELDS))
+        raise ValueError(f"Unknown list field {field!r}. Available fields: {choices}")
+
+    if column != "usage_tags":
+        return sorted(value for value in df[column].dropna().astype(str).unique() if value)
+
+    tags = set()
+    for value in df["usage_tags"].dropna().astype(str):
+        tags.update(tag.strip() for tag in value.split(";") if tag.strip())
+    return sorted(tags)
+
+
+def _normalize_catalog(df: pd.DataFrame) -> pd.DataFrame:
+    df = df.copy()
+    df.columns = [str(column).strip() for column in df.columns]
+
+    required = {
+        "dataset_uid",
+        "dataset_id",
+        "gse_id",
+        "file_path",
+        "usage_tags",
+        "detail_source",
+        "detailed_condition",
+    }
+    missing = sorted(required - set(df.columns))
+    if missing:
+        raise ValueError(f"Metadata catalog is missing required columns: {', '.join(missing)}")
+
+    for column in df.columns:
+        if pd.api.types.is_object_dtype(df[column]):
+            df[column] = df[column].fillna("").astype(str).str.strip()
+
+    return df
+
+
+def _read_metadata_csv(path: Union[str, Path]) -> pd.DataFrame:
+    with Path(path).open("r", encoding="utf-8", newline="") as handle:
+        return pd.read_csv(handle, engine="python")
+
+
+def _as_list(values: Union[str, Iterable[str]]) -> List[str]:
+    if isinstance(values, str):
+        return [values]
+    return [str(value) for value in values]
+
+
+def _normalize_value(value: str, *, case_sensitive: bool) -> str:
+    value = str(value).strip()
+    return value if case_sensitive else value.casefold()
+
+
+def _isin(series: pd.Series, values: Union[str, Iterable[str]], *, case_sensitive: bool) -> pd.Series:
+    normalized_values = {
+        _normalize_value(value, case_sensitive=case_sensitive)
+        for value in _as_list(values)
+    }
+    normalized_series = series.astype(str).map(
+        lambda value: _normalize_value(value, case_sensitive=case_sensitive)
+    )
+    return normalized_series.isin(normalized_values)
+
+
+def _has_usage_tag(series: pd.Series, values: Union[str, Iterable[str]], *, case_sensitive: bool) -> pd.Series:
+    wanted = {
+        _normalize_value(value, case_sensitive=case_sensitive)
+        for value in _as_list(values)
+    }
+
+    def has_tag(value: str) -> bool:
+        tags = {
+            _normalize_value(tag, case_sensitive=case_sensitive)
+            for tag in str(value).split(";")
+            if tag.strip()
+        }
+        return bool(tags & wanted)
+
+    return series.astype(str).map(has_tag)

scmora_db/cli.py

@@ -0,0 +1,204 @@
+"""Command-line interface for scmora-db."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+from typing import List, Optional, Union
+
+from ._version import __version__
+from .catalog import (
+    DEFAULT_MAX_AUTO_MATCHES,
+    DEFAULT_REPO_ID,
+    LISTABLE_FIELDS,
+    list_values,
+    search_datasets,
+)
+from .download import download_datasets
+from .exceptions import AmbiguousDatasetError, TooManyMatchesError
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+    parser = _build_parser()
+    args = parser.parse_args(argv)
+
+    try:
+        return args.func(args)
+    except TooManyMatchesError as exc:
+        print(str(exc), file=sys.stderr)
+        for dataset_uid in exc.dataset_uids:
+            print(dataset_uid)
+        return 2
+    except AmbiguousDatasetError as exc:
+        print(str(exc), file=sys.stderr)
+        for dataset_uid in exc.matches:
+            print(dataset_uid)
+        return 2
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="scmora-db",
+        description="Search, download, and load SCMORA .h5mu datasets.",
+    )
+    parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
+    parser.add_argument("--repo-id", default=DEFAULT_REPO_ID, help="Hugging Face dataset repo ID.")
+    parser.add_argument("--revision", default=None, help="Hugging Face revision, branch, or commit.")
+    parser.add_argument("--token", default=None, help="Hugging Face token for private datasets.")
+    parser.add_argument("--cache-dir", default=None, help="Hugging Face cache directory.")
+    parser.add_argument("--metadata-path", default=None, help="Use a local metadata CSV.")
+    parser.add_argument(
+        "--prefer-remote",
+        action="store_true",
+        help="Download metadata.csv from Hugging Face instead of using bundled metadata.",
+    )
+
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    search_parser = subparsers.add_parser("search", help="Search metadata without downloading .h5mu files.")
+    _add_filter_args(search_parser)
+    search_parser.add_argument("--columns", default=None, help="Comma-separated columns to print.")
+    search_parser.add_argument("--limit", type=int, default=None, help="Maximum rows to print.")
+    search_parser.set_defaults(func=_cmd_search)
+
+    download_parser = subparsers.add_parser("download", help="Download matching .h5mu files.")
+    _add_filter_args(download_parser)
+    download_parser.add_argument("--local-dir", default=None, help="Optional local output directory.")
+    download_parser.add_argument("--force-download", action="store_true", help="Force redownload.")
+    download_parser.add_argument("--max-auto-matches", type=int, default=DEFAULT_MAX_AUTO_MATCHES)
+    download_parser.set_defaults(func=_cmd_download)
+
+    load_parser = subparsers.add_parser("load", help="Download and open matching .h5mu files.")
+    _add_filter_args(load_parser)
+    load_parser.add_argument("--local-dir", default=None, help="Optional local output directory.")
+    load_parser.add_argument("--force-download", action="store_true", help="Force redownload.")
+    load_parser.add_argument("--max-auto-matches", type=int, default=DEFAULT_MAX_AUTO_MATCHES)
+    load_parser.add_argument("--backed", default=None, help='MuData backed mode, for example "r".')
+    load_parser.set_defaults(func=_cmd_load)
+
+    list_parser = subparsers.add_parser("list", help="List available metadata values.")
+    list_parser.add_argument(
+        "field",
+        choices=sorted(LISTABLE_FIELDS),
+    )
+    list_parser.set_defaults(func=_cmd_list)
+
+    return parser
+
+
+def _add_filter_args(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument("--dataset-id", action="append", help="Filter by dataset_id. Can be repeated.")
+    parser.add_argument("--dataset-uid", action="append", help="Filter by unique gse_id/dataset_id.")
+    parser.add_argument("--gse-id", action="append", help="Filter by GSE ID. Can be repeated.")
+    parser.add_argument("--detailed-condition", action="append", help="Filter by detailed_condition.")
+    parser.add_argument("--usage-tag", action="append", help="Filter by one usage tag.")
+    parser.add_argument("--detail-source", action="append", help="Filter by detail_source.")
+    parser.add_argument("--condition", action="append", help="Filter by broad condition.")
+    parser.add_argument("--sample-type", action="append", help="Filter by sample_type.")
+    parser.add_argument("--species", action="append", help="Filter by species.")
+    parser.add_argument("--reference", action="append", help="Filter by reference genome.")
+
+
+def _catalog_kwargs(args) -> dict:
+    return {
+        "repo_id": args.repo_id,
+        "revision": args.revision,
+        "token": args.token,
+        "cache_dir": args.cache_dir,
+        "metadata_path": args.metadata_path,
+        "prefer_remote": args.prefer_remote,
+    }
+
+
+def _filter_kwargs(args) -> dict:
+    return {
+        "dataset_id": args.dataset_id,
+        "dataset_uid": args.dataset_uid,
+        "gse_id": args.gse_id,
+        "detailed_condition": args.detailed_condition,
+        "usage_tag": args.usage_tag,
+        "detail_source": args.detail_source,
+        "condition": args.condition,
+        "sample_type": args.sample_type,
+        "species": args.species,
+        "reference": args.reference,
+    }
+
+
+def _cmd_search(args) -> int:
+    df = search_datasets(**_filter_kwargs(args), **_catalog_kwargs(args))
+    columns = [
+        "dataset_uid",
+        "dataset_id",
+        "gse_id",
+        "detailed_condition",
+        "usage_tags",
+        "detail_source",
+        "file_path",
+    ]
+    if args.columns:
+        columns = [column.strip() for column in args.columns.split(",") if column.strip()]
+    if args.limit is not None:
+        df = df.head(args.limit)
+    print(df.loc[:, columns].to_csv(index=False).rstrip())
+    return 0
+
+
+def _cmd_download(args) -> int:
+    paths = download_datasets(
+        **_filter_kwargs(args),
+        **_catalog_kwargs(args),
+        max_auto_matches=args.max_auto_matches,
+        local_dir=args.local_dir,
+        force_download=args.force_download,
+    )
+    _print_paths(paths)
+    return 0
+
+
+def _cmd_load(args) -> int:
+    from .io import load_datasets
+
+    objects = load_datasets(
+        **_filter_kwargs(args),
+        **_catalog_kwargs(args),
+        max_auto_matches=args.max_auto_matches,
+        local_dir=args.local_dir,
+        force_download=args.force_download,
+        backed=args.backed,
+    )
+    if not isinstance(objects, list):
+        objects = [objects]
+    for obj in objects:
+        print(_summarize_mudata(obj))
+    return 0
+
+
+def _cmd_list(args) -> int:
+    kwargs = _catalog_kwargs(args)
+    values = list_values(args.field, **kwargs)
+
+    for value in values:
+        print(value)
+    return 0
+
+
+def _print_paths(paths: Union[str, List[str]]) -> None:
+    if isinstance(paths, str):
+        print(paths)
+        return
+    for path in paths:
+        print(path)
+
+
+def _summarize_mudata(obj) -> str:
+    n_obs = getattr(obj, "n_obs", "?")
+    n_vars = getattr(obj, "n_vars", "?")
+    mod = getattr(obj, "mod", {})
+    modalities = ",".join(mod.keys()) if hasattr(mod, "keys") else "?"
+    return f"MuData(n_obs={n_obs}, n_vars={n_vars}, modalities={modalities})"
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scmora_db/download.py

@@ -0,0 +1,86 @@
+"""Download .h5mu files from Hugging Face."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import List, Optional, Union
+
+from .catalog import DEFAULT_MAX_AUTO_MATCHES, DEFAULT_REPO_ID, DEFAULT_REPO_TYPE, resolve_matches
+
+
+def download_datasets(
+    *,
+    dataset_id: Optional[Union[str, List[str]]] = None,
+    dataset_uid: Optional[Union[str, List[str]]] = None,
+    gse_id: Optional[Union[str, List[str]]] = None,
+    detailed_condition: Optional[Union[str, List[str]]] = None,
+    usage_tag: Optional[Union[str, List[str]]] = None,
+    detail_source: Optional[Union[str, List[str]]] = None,
+    condition: Optional[Union[str, List[str]]] = None,
+    sample_type: Optional[Union[str, List[str]]] = None,
+    species: Optional[Union[str, List[str]]] = None,
+    reference: Optional[Union[str, List[str]]] = None,
+    repo_id: str = DEFAULT_REPO_ID,
+    revision: Optional[str] = None,
+    token: Optional[Union[str, bool]] = None,
+    cache_dir: Optional[Union[str, Path]] = None,
+    metadata_path: Optional[Union[str, Path]] = None,
+    prefer_remote: bool = False,
+    max_auto_matches: int = DEFAULT_MAX_AUTO_MATCHES,
+    local_dir: Optional[Union[str, Path]] = None,
+    force_download: bool = False,
+) -> Union[str, List[str]]:
+    """Download matching ``.h5mu`` files and return local paths.
+
+    If one dataset matches, a single path string is returned. If two to five
+    datasets match, a list of path strings is returned. More than five matches
+    raises ``TooManyMatchesError``.
+    """
+
+    result = resolve_matches(
+        dataset_id=dataset_id,
+        dataset_uid=dataset_uid,
+        gse_id=gse_id,
+        detailed_condition=detailed_condition,
+        usage_tag=usage_tag,
+        detail_source=detail_source,
+        condition=condition,
+        sample_type=sample_type,
+        species=species,
+        reference=reference,
+        repo_id=repo_id,
+        revision=revision,
+        token=token,
+        cache_dir=cache_dir,
+        metadata_path=metadata_path,
+        prefer_remote=prefer_remote,
+        max_auto_matches=max_auto_matches,
+        require_unique_dataset_id=True,
+    )
+
+    if result.rows.empty:
+        return []
+
+    try:
+        from huggingface_hub import hf_hub_download
+    except ImportError as exc:
+        raise ImportError(
+            "Downloading datasets requires the dependency 'huggingface_hub'. "
+            "Install scmora-db with its default dependencies: pip install scmora-db"
+        ) from exc
+
+    paths = []
+    for _, row in result.rows.iterrows():
+        path = hf_hub_download(
+            repo_id=repo_id,
+            repo_type=DEFAULT_REPO_TYPE,
+            filename=row["file_path"],
+            revision=revision,
+            token=token,
+            cache_dir=cache_dir,
+            local_dir=local_dir,
+            force_download=force_download,
+        )
+        paths.append(path)
+
+    return paths[0] if len(paths) == 1 else paths

scmora_db/exceptions.py

@@ -0,0 +1,34 @@
+"""Custom exceptions for scmora-db."""
+
+
+class ScmoraDbError(Exception):
+    """Base class for scmora-db errors."""
+
+
+class AmbiguousDatasetError(ScmoraDbError):
+    """Raised when a dataset_id query matches multiple datasets."""
+
+    def __init__(self, dataset_id, matches):
+        self.dataset_id = dataset_id
+        self.matches = list(matches)
+        message = (
+            f"dataset_id {dataset_id!r} matched multiple datasets. "
+            "Use dataset_uid or add gse_id to choose one: "
+            + ", ".join(self.matches)
+        )
+        super().__init__(message)
+
+
+class TooManyMatchesError(ScmoraDbError):
+    """Raised when a download or load query matches too many datasets."""
+
+    def __init__(self, count, dataset_uids, limit):
+        self.count = count
+        self.dataset_uids = list(dataset_uids)
+        self.limit = limit
+        ids = ", ".join(self.dataset_uids)
+        message = (
+            f"Query matched {count} datasets, which is more than the automatic "
+            f"limit of {limit}. Matched dataset_uids: {ids}"
+        )
+        super().__init__(message)