fairfetched 0.0.1.post20260304__tar.gz

fairfetched-0.0.1.post20260304/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 lucina-may nollen
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

fairfetched-0.0.1.post20260304/PKG-INFO

@@ -0,0 +1,48 @@
+Metadata-Version: 2.4
+Name: fairfetched
+Version: 0.0.1.post20260304
+Summary: data APIs for reproducible data fetching in cheminformatics
+Project-URL: Homepage, https://github.com/lucinamay/fairfetched
+Requires-Python: >=3.13
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: polars>=1.38.1
+Requires-Dist: pystow>=0.7.28
+Requires-Dist: rich>=14.3.3
+Provides-Extra: rdkit
+Requires-Dist: rdkit>=2025.9.6; extra == "rdkit"
+Provides-Extra: standardise
+Requires-Dist: fairfetched[rdkit]; extra == "standardise"
+Requires-Dist: papyrus-structure-pipeline>=0.0.5; extra == "standardise"
+Provides-Extra: dev
+Requires-Dist: pytest>=9.0.2; extra == "dev"
+Requires-Dist: sqlalchemy>=2.0.46; extra == "dev"
+Requires-Dist: fairfetched[standardise]; extra == "dev"
+Dynamic: license-file
+
+# fairfetched
+data APIs for reproducible data fetching in cheminformatics in line with FAIR principles
+
+# installation 
+you can install this package through
+`uv add fairfetched` (recommended)
+
+or if you do not use the uv package manager:
+`pip install fairfetched`
+
+
+# roadmap
+- [ ] papyrus database support
+  - [x] papyrus latest version download
+  - [x] simple nested filtering
+  - [ ] efficient nested filtering
+  - [ ] all-version support
+  - [ ] built-in pivots
+- [ ] chembl database support
+  - [x] database to tables
+  - [ ] intuitive pre-merged flat files
+  - [ ] database visualisation 
+- [ ] reproducible chembl and papyrus standardisation
+- [ ] automated time-url logging and manifest files
+- [ ] minimisation of dependencies
+- [ ] support for other databases

fairfetched-0.0.1.post20260304/README.md

@@ -0,0 +1,26 @@
+# fairfetched
+data APIs for reproducible data fetching in cheminformatics in line with FAIR principles
+
+# installation 
+you can install this package through
+`uv add fairfetched` (recommended)
+
+or if you do not use the uv package manager:
+`pip install fairfetched`
+
+
+# roadmap
+- [ ] papyrus database support
+  - [x] papyrus latest version download
+  - [x] simple nested filtering
+  - [ ] efficient nested filtering
+  - [ ] all-version support
+  - [ ] built-in pivots
+- [ ] chembl database support
+  - [x] database to tables
+  - [ ] intuitive pre-merged flat files
+  - [ ] database visualisation 
+- [ ] reproducible chembl and papyrus standardisation
+- [ ] automated time-url logging and manifest files
+- [ ] minimisation of dependencies
+- [ ] support for other databases

fairfetched-0.0.1.post20260304/fairfetched/get/__init__.py

@@ -0,0 +1,3 @@
+from .api import Chembl, Papyrus
+
+__all__ = ["Chembl", "Papyrus"]

fairfetched-0.0.1.post20260304/fairfetched/get/api.py

@@ -0,0 +1,127 @@
+from dataclasses import dataclass
+from functools import cached_property
+from pathlib import Path
+from types import ModuleType
+from typing import Tuple
+
+from polars import LazyFrame
+
+import fairfetched.get.chembl as chembl
+import fairfetched.get.papyrus as papyrus
+from fairfetched.utils import BASE_DIR
+
+
+@dataclass(frozen=True)
+class _Base:
+    version: str
+    raw_paths: dict[str, Path]
+    lfs: dict[str, LazyFrame]
+    dir: Path
+    module: ModuleType
+
+    def __hash__(self):
+        return hash(
+            (self.version, str(self.sources), str(self.raw_paths), self.module.__name__)
+        )
+
+    @cached_property
+    def name(self) -> str:
+        return self.module.__name__.split(".")[-1]
+
+    @cached_property
+    def sources(self) -> dict[str, str]:
+        return self.module.get_sources(self.version)
+
+    @property
+    def composed(self) -> dict[str, LazyFrame]:
+        lfs = self.module.compose(self.lfs)
+        return lfs
+
+    @property
+    def bioactivity(self) -> LazyFrame:
+        return self.composed["bioactivity"]
+
+    @property
+    def compounds(self) -> LazyFrame:
+        return self.composed["compounds"]
+
+    @classmethod
+    @cached_property
+    def available_versions(cls) -> Tuple[str]:
+        return cls.module.available_versions()
+
+
+@dataclass(frozen=True)
+class Chembl(_Base):
+    module: ModuleType = chembl
+    extracted_table_paths: dict[str, Path] | None = None
+
+    @cached_property
+    def activity(self) -> LazyFrame:
+        return self.composed["activity"]
+
+    @cached_property
+    def raw_sql_db_path(self) -> Path:
+        return self.raw_paths["sql_db"]
+
+    @classmethod
+    def from_version(
+        cls,
+        version: str | int | float,
+        root_dir: Path | str = f"{BASE_DIR}/chembl",
+    ):
+        """Downloads Chembl for version if not yet present in the given cache directory"""
+        version = chembl.__version_formatter(version)
+        dir = Path(root_dir) / version
+
+        raw_paths: dict[str, Path] = cls.module.ensure_raw(
+            version, cache_dir=dir / "raw"
+        )
+        extracted_table_paths = cls.module.extract_sqlite(
+            raw_paths["sql_db"], cache_dir=dir / "extracted"
+        )
+        lfs = cls.module.clean(extracted_table_paths)
+        return Chembl(
+            version=version,
+            raw_paths=raw_paths,
+            lfs=lfs,
+            dir=dir,
+            module=cls.module,
+            extracted_table_paths=extracted_table_paths,
+        )
+
+    @classmethod
+    def from_latest(cls, root_dir):
+        return cls.from_version(version=cls.module.latest(), root_dir=root_dir)
+
+
+@dataclass(frozen=True)
+class Papyrus(_Base):
+    module: ModuleType = papyrus
+
+    @classmethod
+    def from_version(
+        cls,
+        version: str,
+        root_dir: Path | str = f"{BASE_DIR}/papyrus",
+    ):
+        """Downloads Chembl for version if not yet present in the given cache directory"""
+        dir = Path(root_dir) / version
+        raw_paths: dict[str, Path] = cls.module.ensure_raw(
+            version, cache_dir=dir / "raw"
+        )
+        lfs = cls.module.clean(raw_paths)
+        return Papyrus(
+            version=version,
+            raw_paths=raw_paths,
+            lfs=lfs,
+            dir=dir,
+            module=cls.module,
+        )
+
+    @classmethod
+    def from_latest(
+        cls,
+        root_dir: Path | str = f"{BASE_DIR}/papyrus",
+    ):
+        return cls.from_version(version=cls.module.latest(), root_dir=root_dir)

fairfetched-0.0.1.post20260304/fairfetched/get/chembl.py

@@ -0,0 +1,195 @@
+from pathlib import Path
+from typing import Any
+
+import polars as pl
+
+from fairfetched.utils import (
+    BASE_DIR,
+    ensure_url,
+    file_suffix_from_url,
+    lowercase_columns,
+    sqlite_db_to_parquets,
+    untar_sqlite,
+)
+from fairfetched.utils.typing import ComposedLFDict
+
+CHEMBL_DIR = BASE_DIR / "chembl"
+
+
+def __version_formatter(version: int | float | str) -> str:
+    if isinstance(version, int | float):
+        version = str(version)
+    if not isinstance(version, str):
+        try:
+            version = str(version)
+        finally:
+            raise TypeError(f"invalid version type: {type(version)}")
+
+    version = version.lstrip("0")
+    # for canonicalize the version number 22.1 and 24.1 and left pad with a zero if needed
+    return version.replace(".", "_").zfill(2)
+
+
+def _version_to_url(version: str):
+    base = "https://ftp.ebi.ac.uk/pub/databases/chembl/ChEMBLdb/releases"
+    return f"{base}/chembl_{version}/chembl_{version}"
+
+
+CHEMBL_VERSIONS: dict[str, dict[str, str]] = {
+    version: {"sql_db": _version_to_url(version)}
+    for version in sorted(
+        map(__version_formatter, list(range(1, 37)) + ["24_1", "22_1"])
+    )
+}
+
+
+def available_versions() -> tuple[str, ...]:
+    return tuple(CHEMBL_VERSIONS.keys())
+
+
+def latest() -> str:
+    return available_versions()[-1]
+
+
+def get_sources(version: str) -> dict[str, str]:
+    return CHEMBL_VERSIONS[str(version)]
+
+
+def ensure_raw(version: str, cache_dir: Path | str | None = None) -> dict[str, Path]:
+    """downloads the original sql database, with its original name and compression"""
+    if cache_dir is None:
+        cache_dir = CHEMBL_DIR / version
+    cache_dir = Path(cache_dir)
+    return {
+        name: ensure_url(url=url, path=cache_dir / f"{file_suffix_from_url(url)}")
+        for name, url in get_sources(version).items()
+    }
+
+
+def extract_sqlite(
+    sql_tar_gz_path: Path | str, cache_dir: Path | str | Any | None = None
+) -> dict[str, Path]:
+    if cache_dir is None:
+        cache_dir = Path(sql_tar_gz_path).parent / "extracted"
+
+    raw_sql = untar_sqlite(sql_tar_gz_path)
+    parquets = sqlite_db_to_parquets(raw_sql, cache_dir=cache_dir)
+    return parquets
+
+
+def clean(extracted_sqlite_parquet_paths: dict[str, Path]) -> dict[str, pl.LazyFrame]:
+    return {
+        name: pl.scan_parquet(path_)
+        .pipe(lowercase_columns)
+        .fill_nan(None)
+        .with_columns(
+            pl.col(pl.String).replace({"": None}),
+        )
+        for name, path_ in extracted_sqlite_parquet_paths.items()
+    }
+
+
+def compose(lfs: dict[str, pl.LazyFrame]) -> ComposedLFDict:
+    """Join/combine lazy frames. Optional, returns single LF."""
+    return {
+        "bioactivity": _bioactivities(lfs),
+        "compounds": _compounds(lfs),
+        "proteins": lfs["protein"],
+        "components": _components(lfs),
+    }
+
+
+def _bioactivities(lfs) -> pl.LazyFrame:
+    return (
+        # dfs["activity_properties"]
+        # .join(dfs["activities"], on="activity_id", how="left")
+        # .join(dfs["action_type"], on="action_type", how="left")
+        # .join
+        lfs["bioactivity"]
+        .join(
+            lfs["protein"],
+            on="target_id",
+            how="left",
+            maintain_order="left",
+            validate="m:1",  # one unique protein only from right, can reoccur within compounds.
+        )
+        .join(lfs["action_type"], on="action_type", how="left", suffix="_action_type")
+        .join(
+            lfs["assays"], on="assay_id", how="left", suffix="_assay"
+        )  # doc_ids to this
+        .join(lfs["assay_type"], on="assay_type", how="left", suffix="_assay_type")
+        .join(lfs, on="bao_format")
+        # .join(dfs[])
+    )
+
+
+def _components(lfs) -> pl.LazyFrame:
+    return (
+        lfs["component_sequences"]
+        .join(
+            lfs["component_class"],
+            on="component_id",
+            how="left",
+            suffix="_class",
+            validate="1:m",
+        )
+        .join(
+            lfs["component_domains"].join(
+                lfs["domains"],
+                on="domain_id",
+                how="left",
+                suffix="_domains",
+                validate="m:1",
+            ),
+            on="component_id",
+            how="left",
+            suffix="_domains",
+            validate="1:m",
+        )
+        # .join(
+        #     dfs["component_synonyms"],
+        #     on="component_id",
+        #     how="left",
+        #     suffix="_synonyms",
+        #     validate="m:m",
+        # )
+    )
+
+
+def _compounds(lfs) -> pl.LazyFrame:
+    return (
+        lfs["molecule_dictionary"]
+        .join(
+            lfs["compound_properties"],
+            on="molregno",
+            how="left",
+            suffix="_compound_properties",
+            validate="1:1",
+        )
+        .join(
+            lfs["compound_structures"],
+            on="molregno",
+            how="left",
+            suffix="_compound_structures",
+            validate="1:1",
+        )
+        .join(
+            lfs["compound_records"].join(
+                lfs["docs"], on="doc_id", how="left", suffix="_doc", validate="m:1"
+            ),
+            on="molregno",
+            how="left",
+            suffix="_compound_records",
+            validate="1:m",
+        )
+        .join(
+            lfs["compound_structural_alerts"],
+            #     .join(
+            #     dfs["docs"], on="doc_id", how="left", suffix="_doc",validate="m:1"
+            # )
+            on="molregno",
+            how="left",
+            suffix="_compound_structural_alerts",
+            validate="1:m",
+        )
+    )

fairfetched-0.0.1.post20260304/fairfetched/get/papyrus.py

@@ -0,0 +1,130 @@
+"""Papyrus dataset utilities for downloading, cleaning, and joining bioactivity and protein data.
+
+This module provides functions to ensure the presence of raw and cleaned Papyrus dataset files,
+and defines the Papyrus_57 database configuration.
+"""
+
+from pathlib import Path
+from typing import Any
+
+import polars as pl
+
+from fairfetched.utils import (
+    BASE_DIR,
+    ensure_url,
+    file_suffix_from_url,
+    lowercase_columns,
+    scan_tsvxz,
+)
+from fairfetched.utils.typing import ComposedLFDict
+
+PAPYRUS_VERSIONS: dict[str, dict[str, str]] = {
+    "05.7": {
+        "bioactivity": "https://zenodo.org/records/13987985/files/05.7_combined_set_with_stereochemistry.tsv.xz?download=1",
+        # "bioactivity_nostereochemistry": "https://zenodo.org/records/13987985/files/05.7_combined_set_without_stereochemistry.tsv.xz?download=1",
+        "readme": "https://zenodo.org/records/13987985/files/README.txt?download=1",
+        "protein": "https://zenodo.org/records/13987985/files/05.7_combined_set_protein_targets.tsv.xz?download=1",
+    },
+    "05.6": {
+        "bioactivity": "https://zenodo.org/records/7373214/files/05.6_combined_set_with_stereochemistry.tsv.xz?download=1",
+        # "bioactivity_nostereochemistry": "https://zenodo.org/records/7373214/files/05.6_combined_set_without_stereochemistry.tsv.xz?download=1",
+        "readme": "https://zenodo.org/records/7373214/files/README.txt?download=1",
+        "protein": "https://zenodo.org/records/7373214/files/05.6_combined_set_protein_targets.tsv.xz?download=1",
+    },
+}
+
+
+def available_versions() -> tuple[str, ...]:
+    return tuple(PAPYRUS_VERSIONS.keys())
+
+
+def latest() -> str:
+    return available_versions()[-1]
+
+
+def get_sources(version: str) -> dict[str, str]:
+    return PAPYRUS_VERSIONS[str(version)]
+
+
+def ensure_raw(
+    version: str, cache_dir: Path | str | Any | None = None
+) -> dict[str, Path]:
+    """Download if missing, return path to raw file."""
+    if cache_dir is None:
+        cache_dir = BASE_DIR / "papyrus" / version
+    cache_dir = Path(cache_dir)
+
+    return {
+        name: ensure_url(url=url, path=cache_dir / f"{file_suffix_from_url(url)}")
+        for name, url in get_sources(version).items()
+    }
+
+
+def clean(raw_filepath_dict: dict[str, Path]) -> dict[str, pl.LazyFrame]:
+    """Transform raw → clean, return lazy frames. No I/O."""
+    schema_overrides = {
+        "Year": pl.Int32,
+        "pchembl_value_Mean": pl.Float64,
+        "pchembl_value_StdDev": pl.Float64,
+        "pchembl_value_SEM": pl.Float64,
+        "pchembl_value_N": pl.Float64,
+        "pchembl_value_Median": pl.Float64,
+        "pchembl_value_MAD": pl.Float64,
+    }
+
+    return {
+        "protein": (
+            scan_tsvxz(raw_filepath_dict["protein"], separator="\t")
+            .pipe(lowercase_columns)
+            .rename({"uniprotid": "uniprot_id"})
+        ),
+        "bioactivity": (
+            scan_tsvxz(
+                raw_filepath_dict["bioactivity"],
+                separator="\t",
+                infer_schema=False,
+                schema_overrides=schema_overrides,
+                null_values="NA",  # which values to take as None
+            )
+            .pipe(lowercase_columns)
+            .cast({"pchembl_value_n": pl.Int64})
+        ),
+    }
+
+
+def compose(lfs: dict[str, pl.LazyFrame]) -> ComposedLFDict:
+    """Join/combine lazy frames. Optional, returns single LF."""
+    return {
+        "bioactivity": lfs["bioactivity"].join(
+            lfs["protein"],
+            on="target_id",
+            how="left",
+            maintain_order="left",
+            validate="m:1",  # one unique protein only from right, can reoccur within compounds.
+        ),
+        "compounds": lfs["bioactivity"]
+        .drop(
+            "activity_id",
+        )
+        .unique(("connectivity", "inchikey", "inchi")),
+        "full": lfs["bioactivity"].join(
+            lfs["protein"],
+            on="target_id",
+            how="left",
+            maintain_order="left",
+            validate="m:1",  # one unique protein only from right, can reoccur within compounds.
+        ),
+    }
+
+
+def help() -> None:
+    """prints out example usage"""
+    print("""
+        Example usage:
+        ```
+        raw = ensure_raw("05.7")
+        lfs = clean(raw)
+        final_lf = compose(lfs)
+        final_lf.sink_parquet("output.parquet")
+        ```
+        """)

fairfetched-0.0.1.post20260304/fairfetched/utils/__init__.py

@@ -0,0 +1,14 @@
+from .ensure import ensure_url
+from .files import file_suffix_from_url, untar_sqlite
+from .polars import lowercase_columns, scan_tsvxz, sqlite_db_to_parquets
+from .storage import BASE_DIR
+
+__all__ = [
+    "BASE_DIR",
+    ensure_url,
+    lowercase_columns,
+    scan_tsvxz,
+    file_suffix_from_url,
+    sqlite_db_to_parquets,
+    untar_sqlite,
+]

fairfetched-0.0.1.post20260304/fairfetched/utils/_optional.py

@@ -0,0 +1,30 @@
+try:
+    from tqdm import (  # ty: ignore[unresolved-import,unused-ignore-comment] # ty: ignore[unused-ignore-comment]
+        tqdm as tqdm_track,
+    )
+except ImportError:
+
+    def tqdm_track(iterable, *args, **kwargs):
+        return iter(iterable)
+
+
+try:
+    from rich.progress import (  # ty: ignore[unresolved-import,unused-ignore-comment] # ty: ignore[unused-ignore-comment]
+        track as rich_track,
+    )
+
+except ImportError:
+
+    def rich_track(iterable, *args, **kwargs):
+        return iter(iterable)
+
+
+try:
+    from marimo import (  # ty: ignore[unresolved-import,unused-ignore-comment] # ty: ignore[unused-ignore-comment]
+        running_in_notebook as in_marimo,
+    )
+
+except ImportError:
+
+    def in_marimo():
+        return False

fairfetched-0.0.1.post20260304/fairfetched/utils/_track.py

@@ -0,0 +1,31 @@
+from typing import Iterable
+
+from _optional import in_marimo, rich_track, tqdm_track
+
+
+def track(iterable: Iterable, desc: str = "", total: int | None = None):
+    """Progress bar that adapts to the environment (marimo or terminal)."""
+    if not total:
+        try:
+            total = len(iterable)  # ty: ignore[invalid-argument-type]
+        except TypeError:
+            pass
+    if in_marimo():
+        # Use tqdm with thin bar characters for a slimmer look
+        return tqdm_track(
+            iterable,
+            desc=desc,
+            total=total,
+            bar_format="{desc} {bar:20} {percentage:3.0f}% {n_fmt}/{total_fmt} [{elapsed}<{remaining}, {rate_fmt}]",
+            ascii="─━",
+            leave=True,
+        )
+    return rich_track(
+        iterable,
+        description=desc,
+        total=total,
+        transient=False,
+        style="green",
+        complete_style="bold green",
+        console=None,
+    )

fairfetched-0.0.1.post20260304/fairfetched/utils/ensure.py

@@ -0,0 +1,30 @@
+import datetime
+import logging as lg
+from pathlib import Path
+
+import requests
+
+from ._track import track
+
+
+def ensure_url(url: str, path: Path, force: bool = False) -> Path:
+    """Downloads url to path if not already existing. Makes path dirs if not existing"""
+    path.parent.mkdir(parents=True, exist_ok=True)
+
+    if path.exists() and not force:
+        lg.debug(f"File already exists at {path}. Skipping download.")
+        return path
+
+    response = requests.get(url)
+    response.raise_for_status()
+    with open(path, "wb") as f:
+        total = int(response.headers.get("content-length", 0))
+        chunk_size = 8192
+        for chunk in track(
+            response.iter_content(chunk_size=chunk_size),
+            total=(total // chunk_size) + int(total % chunk_size != 0),
+            desc=f"downloading {url.split('/')[-1]}",
+        ):
+            f.write(chunk)
+    lg.info(f"Downloaded {url} to {path} on {datetime.datetime.now()}")
+    return path