fairfetched 0.0.1.dev202607271__py3-none-any.whl

fairfetched/get/__init__.py

@@ -0,0 +1,9 @@
+from .dataset import Chembl, Papyrus
+
+__all__ = ["Chembl", "Papyrus"]
+
+if __name__ == "__main__":
+    from .papyrus import ensure_raw_files, latest
+
+    print(latest())
+    ensure_raw_files(latest())

fairfetched/get/chembl.py

@@ -0,0 +1,219 @@
+from collections.abc import Sequence
+from pathlib import Path
+from typing import Any
+
+import polars as pl
+
+from fairfetched.utils import (
+    BASE_DIR,
+    ensure_sqlite_db_to_parquets,
+    ensure_untarred_sqlite,
+    ensure_url,
+    file_suffix_from_url,
+    lowercase_columns,
+)
+from fairfetched.utils.typing import BioactivityDBViews
+
+CHEMBL_DIR = BASE_DIR / "chembl"
+
+
+def _format_version(version: float | str) -> str:
+    if isinstance(version, int | float):
+        version = str(version)
+    if isinstance(version, Sequence) and not isinstance(version, str):
+        raise TypeError(f"invalid version type: {type(version)}")
+    if not isinstance(version, str):
+        try:
+            version = str(version)
+        except Exception:
+            raise TypeError(f"invalid version type: {type(version)}")
+
+    version = version.lstrip("0")
+    if "." in version:
+        version = version.split(".")[0].zfill(2) + "." + version.split(".")[1]
+    # for canonicalize the version number 22.1 and 24.1 and left pad with a zero if needed
+    return version.replace(".", "_").replace("_0", "").zfill(2)
+
+
+def _version_to_url(version: str):
+    base = "https://ftp.ebi.ac.uk/pub/databases/chembl/ChEMBLdb/releases"
+    return f"{base}/chembl_{version}/chembl_{version}_sqlite.tar.gz"
+
+
+CHEMBL_VERSIONS: dict[str, dict[str, str]] = {
+    version: {"sql_db": _version_to_url(version)}
+    for version in sorted(map(_format_version, list(range(1, 38)) + ["24_1", "22_1"]))
+}
+
+
+def available_versions() -> tuple[str, ...]:
+    return tuple(CHEMBL_VERSIONS.keys())
+
+
+def latest() -> str:
+    return available_versions()[-1]
+
+
+def source_urls(version: str) -> dict[str, str]:
+    return CHEMBL_VERSIONS[str(version)]
+
+
+def ensure_raw_files(
+    version: str, raw_dir: Path | str | None = None, force=False
+) -> dict[str, Path]:
+    """Download the original SQL database with its original name and compression."""
+    if raw_dir is None:
+        raw_dir = CHEMBL_DIR / version
+    raw_dir = Path(raw_dir)
+    return {
+        name: ensure_url(
+            url=url, path=raw_dir / f"{name}{file_suffix_from_url(url)}", force=force
+        )
+        for name, url in source_urls(version).items()
+    }
+
+
+def ensure_parquet_tables(
+    raw_paths: dict[str, Path], table_dir: Path | str | Any | None = None
+) -> dict[str, Path]:
+    sql_tar_gz_path = raw_paths["sql_db"]
+    if table_dir is None:
+        table_dir = Path(sql_tar_gz_path).parent / "extracted"
+    table_dir = Path(table_dir)
+    table_dir.mkdir(exist_ok=True, parents=True)
+
+    raw_sql = ensure_untarred_sqlite(sql_tar_gz_path)
+    # the untarred should also stay so that we have access.....
+    # #@TODO: perhaps make tables deterministic for chembl to circumvent
+    parquets = ensure_sqlite_db_to_parquets(raw_sql, cache_dir=table_dir, force=False)
+
+    return parquets
+
+
+def cleanly_scan_parquet(path_: Path | str) -> pl.LazyFrame:
+    """scans parquet paths and lazily handles null value conversion to None"""
+    return (
+        pl.scan_parquet(path_)
+        .pipe(lowercase_columns)
+        .fill_nan(None)
+        .with_columns(
+            pl.col(pl.String).replace({"": None}),
+        )
+    )
+
+
+def cleanly_scan_parquet_tables(
+    parquet_paths: dict[str, Path],
+) -> dict[str, pl.LazyFrame]:
+    """scans parquet paths and lazily handles null value conversion to None"""
+    return {name: cleanly_scan_parquet(path_) for name, path_ in parquet_paths.items()}
+
+
+# def build_views(lfs: dict[str, pl.LazyFrame]) -> ComposedLFDict:
+def build_views(parquet_paths: dict[str, Path]) -> BioactivityDBViews:
+    """Build joined domain views from the scanned source tables."""
+    return {
+        "bioactivity": _bioactivities(parquet_paths),
+        "compounds": _compounds(parquet_paths),
+        "proteins": cleanly_scan_parquet(parquet_paths["protein"]),
+        "components": _components(parquet_paths),
+    }
+
+
+def _bioactivities(parquet_paths: dict[str, Path]) -> pl.LazyFrame:
+    lfs = cleanly_scan_parquet_tables(parquet_paths)
+    return (
+        # dfs["activity_properties"]
+        # .join(dfs["activities"], on="activity_id", how="left")
+        # .join(dfs["action_type"], on="action_type", how="left")
+        # .join
+        lfs["bioactivity"]
+        .join(
+            lfs["protein"],
+            on="target_id",
+            how="left",
+            maintain_order="left",
+            validate="m:1",  # one unique protein only from right, can reoccur within compounds.
+        )
+        .join(lfs["action_type"], on="action_type", how="left", suffix="_action_type")
+        .join(
+            lfs["assays"], on="assay_id", how="left", suffix="_assay"
+        )  # doc_ids to this
+        .join(lfs["assay_type"], on="assay_type", how="left", suffix="_assay_type")
+        # .join(lfs, on="bao_format")
+        # .join(dfs[])
+    )
+
+
+def _components(parquet_paths: dict[str, Path]) -> pl.LazyFrame:
+    lfs = cleanly_scan_parquet_tables(parquet_paths)
+    return (
+        lfs["component_sequences"]
+        .join(
+            lfs["component_class"],
+            on="component_id",
+            how="left",
+            suffix="_class",
+            validate="1:m",
+        )
+        .join(
+            lfs["component_domains"].join(
+                lfs["domains"],
+                on="domain_id",
+                how="left",
+                suffix="_domains",
+                validate="m:1",
+            ),
+            on="component_id",
+            how="left",
+            suffix="_domains",
+            validate="1:m",
+        )
+        # .join(
+        #     dfs["component_synonyms"],
+        #     on="component_id",
+        #     how="left",
+        #     suffix="_synonyms",
+        #     validate="m:m",
+        # )
+    )
+
+
+def _compounds(parquet_paths: dict[str, Path]) -> pl.LazyFrame:
+    lfs = cleanly_scan_parquet_tables(parquet_paths)
+    return (
+        lfs["molecule_dictionary"]
+        .join(
+            lfs["compound_properties"],
+            on="molregno",
+            how="left",
+            suffix="_compound_properties",
+            validate="1:1",
+        )
+        .join(
+            lfs["compound_structures"],
+            on="molregno",
+            how="left",
+            suffix="_compound_structures",
+            validate="1:1",
+        )
+        .join(
+            lfs["compound_records"].join(
+                lfs["docs"], on="doc_id", how="left", suffix="_doc", validate="m:1"
+            ),
+            on="molregno",
+            how="left",
+            suffix="_compound_records",
+            validate="1:m",
+        )
+        .join(
+            lfs["compound_structural_alerts"],
+            #     .join(
+            #     dfs["docs"], on="doc_id", how="left", suffix="_doc",validate="m:1"
+            # )
+            on="molregno",
+            how="left",
+            suffix="_compound_structural_alerts",
+            validate="1:m",
+        )
+    )

fairfetched/get/dataset.py

@@ -0,0 +1,167 @@
+from dataclasses import dataclass
+from functools import cached_property
+from pathlib import Path
+
+from polars import LazyFrame
+
+from fairfetched.get import chembl, papyrus
+from fairfetched.utils import BASE_DIR
+from fairfetched.utils.typing import BioactivityDBViews, DatasetGetModule
+
+
+@dataclass(frozen=True)
+class _Base:
+    """lightweight wrapper serving as the main point for a united API per database"""
+
+    version: str
+    raw_paths: dict[str, Path]
+    parquet_paths: dict[str, Path]
+    dir: Path
+    module: DatasetGetModule
+
+    def __str__(self) -> str:
+        return f"{self.name}_{self.version}"
+
+    def __repr__(self) -> str:
+        return f"<{self.name.capitalize()}_{self.version} at {self.dir}"
+
+    def __hash__(self):
+        return hash(
+            (self.version, str(self.sources), str(self.raw_paths), self.module.__name__)
+        )
+
+    @cached_property
+    def name(self) -> str:
+        return self.module.__name__.split(".")[-1]
+
+    @cached_property
+    def sources(self) -> dict[str, str]:
+
+        sources = self.module.source_urls(self.version)
+        return sources
+
+    @property
+    def lfs(self) -> dict[str, LazyFrame]:
+        return self.module.cleanly_scan_parquet_tables(self.parquet_paths)
+
+    @property
+    def views(self) -> BioactivityDBViews:
+        return self.module.build_views(self.parquet_paths)
+
+    @property
+    def bioactivity(self) -> LazyFrame:
+        return self.views["bioactivity"]
+
+    @property
+    def compounds(self) -> LazyFrame:
+        return self.views["compounds"]
+
+    @classmethod
+    def available_versions(cls) -> tuple[str, ...]:
+        return cls.module.available_versions()
+
+
+@dataclass(frozen=True)
+class Chembl(_Base):
+    module: DatasetGetModule = chembl
+
+    @staticmethod
+    def get_available_versions():
+        return chembl.available_versions()
+
+    @cached_property
+    def activity(self) -> LazyFrame:
+        return self.views["bioactivity"]
+
+    @cached_property
+    def raw_sql_db_path(self) -> Path:
+        return self.raw_paths["sql_db"]
+
+    @classmethod
+    def from_version(
+        cls,
+        version: str | int | float,  # ruff: ignore[PYI041]
+        root_dir: Path | str = f"{BASE_DIR}/chembl",
+        force: bool = False,
+    ) -> "Chembl":
+        """Downloads Chembl for version if not yet present in the given cache directory"""
+        version = chembl._format_version(version)
+        dir = Path(root_dir) / version
+
+        raw_paths: dict[str, Path] = chembl.ensure_raw_files(
+            version, raw_dir=dir / "raw", force=force
+        )
+
+        parquet_paths = chembl.ensure_parquet_tables(
+            raw_paths, table_dir=dir / "parquet"
+        )
+        return Chembl(
+            version=version,
+            raw_paths=raw_paths,
+            parquet_paths=parquet_paths,
+            dir=dir,
+            module=cls.module,
+        )
+
+    @classmethod
+    def from_latest(
+        cls,
+        root_dir: Path | str = f"{BASE_DIR}/chembl",
+        force: bool = False,
+    ) -> "Chembl":
+        return cls.from_version(version=chembl.latest(), root_dir=root_dir, force=force)
+
+
+@dataclass(frozen=True)
+class Papyrus(_Base):
+    module: DatasetGetModule = papyrus
+
+    @staticmethod
+    def get_available_versions():
+        return papyrus.available_versions()
+
+    @property
+    def proteins(self) -> LazyFrame:
+        return self.views["proteins"]
+
+    @property
+    def full_data(self) -> LazyFrame:
+        """all data (bioactivity + protein data),
+        composed into one flat tabular format LazyFrame"""
+        return self.views["full"]
+
+    @classmethod
+    def from_version(
+        cls,
+        version: str,
+        root_dir: Path | str = f"{BASE_DIR}/papyrus",
+    ) -> "Papyrus":
+        """Downloads Chembl for version if not yet present in the given cache directory"""
+        dir = Path(root_dir) / version
+        raw_paths: dict[str, Path] = papyrus.ensure_raw_files(
+            version, raw_dir=dir / "raw"
+        )
+        parquet_paths: dict[str, Path] = papyrus.ensure_parquet_tables(
+            raw_paths, table_dir=dir / "parquet"
+        )
+        return Papyrus(
+            version=version,
+            raw_paths=raw_paths,
+            parquet_paths=parquet_paths,
+            dir=dir,
+            module=cls.module,
+        )
+
+    @classmethod
+    def from_latest(
+        cls,
+        root_dir: Path | str = f"{BASE_DIR}/papyrus",
+    ) -> "Papyrus":
+        return cls.from_version(version=papyrus.latest(), root_dir=root_dir)
+
+
+if __name__ == "__main__":
+    p = Papyrus.from_latest()
+    p.views["proteins"]
+
+    p.lfs["proteins"]

fairfetched/get/papyrus.py

@@ -0,0 +1,156 @@
+"""Papyrus dataset utilities for downloading, cleaning, and joining bioactivity and protein data.
+
+This module provides functions to ensure the presence of raw and cleaned Papyrus dataset files,
+and defines the Papyrus_57 database configuration.
+"""
+
+from pathlib import Path
+from typing import Any
+
+import polars as pl
+
+from fairfetched.utils import (
+    BASE_DIR,
+    decompress_and_scan_tsvxz,
+    ensure_url,
+    file_suffix_from_url,
+    lowercase_columns,
+)
+from fairfetched.utils.typing import BioactivityDBViews
+
+PAPYRUS_VERSIONS: dict[str, dict[str, str]] = {
+    "05.6": {
+        "bioactivity": "https://zenodo.org/records/7373214/files/05.6_combined_set_with_stereochemistry.tsv.xz",
+        # "bioactivity_nostereochemistry": "https://zenodo.org/records/7373214/files/05.6_combined_set_without_stereochemistry.tsv.xz",
+        "readme": "https://zenodo.org/records/7373214/files/README.txt",
+        "protein": "https://zenodo.org/records/7373214/files/05.6_combined_set_protein_targets.tsv.xz",
+    },
+    "05.7": {
+        "bioactivity": "https://zenodo.org/records/13987985/files/05.7_combined_set_with_stereochemistry.tsv.xz",
+        # "bioactivity_nostereochemistry": "https://zenodo.org/records/13987985/files/05.7_combined_set_without_stereochemistry.tsv.xz",
+        "readme": "https://zenodo.org/records/13987985/files/README.txt",
+        "protein": "https://zenodo.org/records/13987985/files/05.7_combined_set_protein_targets.tsv.xz",
+    },
+}
+
+
+def available_versions() -> tuple[str, ...]:
+    return tuple(PAPYRUS_VERSIONS.keys())
+
+
+def latest() -> str:
+    return available_versions()[-1]
+
+
+def source_urls(version: str) -> dict[str, str]:
+    return PAPYRUS_VERSIONS[str(version)]
+
+
+def ensure_raw_files(
+    version: str, raw_dir: Path | str | Any | None = None
+) -> dict[str, Path]:
+    """Download if missing, return path to raw file."""
+    if raw_dir is None:
+        raw_dir = BASE_DIR / "papyrus" / version / "raw"
+    raw_dir = Path(raw_dir)
+
+    return {
+        name: ensure_url(url=url, path=raw_dir / f"{name}{file_suffix_from_url(url)}")
+        for name, url in source_urls(version).items()
+    }
+
+
+def ensure_parquet_tables(
+    raw_paths: dict[str, Path],
+    table_dir: Path | str | None = None,
+) -> dict[str, Path]:
+    """Downloads if missing, extracts to streamable parquets for lazy loading,
+    returns paths to raw files and parquet files. Ignores README.txt file
+
+    parquet_dir default is raw_filepath_dir.parent / "parquet"
+
+    """
+    if table_dir is None:
+        table_dir = next(iter(raw_paths.values())).parent.parent / "parquet"
+    table_dir = Path(table_dir)
+    table_dir.mkdir(exist_ok=True, parents=True)
+    schema_overrides = {
+        "Year": pl.Int32,
+        "pchembl_value_Mean": pl.Float64,
+        "pchembl_value_StdDev": pl.Float64,
+        "pchembl_value_SEM": pl.Float64,
+        # because of stray floats in pchembl_value_N, we do float -> int
+        "pchembl_value_N": pl.Float64,
+        "pchembl_value_Median": pl.Float64,
+        "pchembl_value_MAD": pl.Float64,
+    }
+
+    filepath_dict = {}
+    for name, path_ in raw_paths.items():
+        if path_.name.lower() == "readme.txt":
+            continue
+        new_path = table_dir / f"{path_.stem.split('.')[0]}.parquet"
+        filepath_dict[name] = new_path
+        if new_path.exists():
+            continue
+        decompress_and_scan_tsvxz(
+            path_,
+            separator="\t",
+            infer_schema=False,
+            schema_overrides=schema_overrides,
+            null_values=["NA", ""],  # which values to take as None
+        ).cast(
+            {"pchembl_value_N": pl.Int64} if name == "bioactivity" else {}
+        ).sink_parquet(new_path)
+    return filepath_dict
+
+
+def cleanly_scan_parquet_tables(
+    parquet_paths: dict[str, Path],
+) -> dict[str, pl.LazyFrame]:
+    """Scan table Parquet files and apply dataset-specific column cleanup."""
+
+    return {
+        "protein": (
+            pl.scan_parquet(parquet_paths["protein"])
+            .pipe(lowercase_columns)
+            .rename({"uniprotid": "uniprot_id"})
+        ),
+        "bioactivity": (
+            pl.scan_parquet(parquet_paths["bioactivity"]).pipe(lowercase_columns)
+        ),
+    }
+
+
+def build_views(parquet_paths: dict[str, Path]) -> BioactivityDBViews:
+    """Build joined domain views from the scanned source tables."""
+    lfs = cleanly_scan_parquet_tables(parquet_paths)
+    return {
+        "bioactivity": lfs["bioactivity"],
+        "compounds": lfs["bioactivity"]
+        .drop(
+            "activity_id",
+        )
+        .unique(("connectivity", "inchikey", "inchi")),
+        "full": lfs["bioactivity"].join(
+            lfs["protein"],
+            on="target_id",
+            how="left",
+            maintain_order="left",
+            validate="m:1",  # one unique protein only from right, can reoccur within compounds.
+        ),
+        "proteins": lfs["protein"],
+    }
+
+
+def help() -> None:
+    """prints out example usage"""
+    print("""
+        Example usage:
+        ```
+        raw = ensure_raw_files("05.7")
+        tables = cleanly_scan_parquet_tables(raw)
+        views = build_views(tables)
+        views["full"].sink_parquet("output.parquet")
+        ```
+        """)

fairfetched/standardize/___mol_expr.pyi

@@ -0,0 +1,23 @@
+# mol_expr.pyi
+import polars as pl
+
+from .mol_expr import MolExprs, MolFn
+
+# class MolExpr:
+#     def __init__(self, expr: pl.Expr) -> None: ...
+#     def from_smiles(self) -> pl.Expr: ...
+#     def from_inchi(self) -> pl.Expr: ...
+#     def to_smiles(self) -> pl.Expr: ...
+#     def to_inchi(self) -> pl.Expr: ...
+#     def to_inchikey(self) -> pl.Expr: ...
+#     def to_kekulised_smiles(self) -> pl.Expr: ...
+#     def num_atoms(self) -> pl.Expr: ...
+#     def num_heavy_atoms(self) -> pl.Expr: ...
+#     def standardise(self, *steps: MolFn) -> pl.Expr: ...
+
+# teach ty that pl.Expr has a .mol accessor
+class _ExprMolAccessor:
+    mol: MolExprs
+
+# ty picks this up via the __getattr__ extension pattern
+def __getattr__(name: str) -> MolExprs: ...

fairfetched/standardize/_optional.py

@@ -0,0 +1,56 @@
+import logging as lg
+
+try:
+    from chembl_structure_pipeline import (  # ty:ignore[unresolved-import] #ty:ignore[unused-ignore-comment] #ty:ignore[unused-ignore-comment]
+        get_parent_mol as chembl_get_parent_mol,
+    )
+    from chembl_structure_pipeline import (  # ty:ignore[unresolved-import] #ty:ignore[unused-ignore-comment] #ty:ignore[unused-ignore-comment]
+        standardize_mol as chembl_standardize,
+    )
+except ImportError as e:
+    lg.warning("""
+        you should install chembl_structure_pipeline if you want
+        to use chembl standardisation
+        """)
+    CHEMBL_IMPORT_ERROR = e
+
+    def chembl_get_parent_mol(mol):
+        """placeholder for chembl_structure_pipeline.get_parent_mol in case
+        the import doesn't work.
+        raises respective import error when called"""
+        lg.warning("""
+            please install `chembl_structure_pipeline` if you want to use
+            ChEMBL standardisation
+            """)
+        raise CHEMBL_IMPORT_ERROR
+
+    def chembl_standardize(mol):
+        """placeholder for chembl_structure_pipeline.chembl_standardize in case
+        the import doesn't work.
+
+        raises respective import error when called.
+        """
+        lg.warning("""
+            please install `chembl_structure_pipeline` if you want to use
+            ChEMBL standardisation
+            """)
+        raise CHEMBL_IMPORT_ERROR
+
+
+try:
+    from papyrus_structure_pipeline import (  # ty:ignore[unresolved-import] #ty:ignore[unused-ignore-comment] #ty:ignore[unused-ignore-comment]
+        standardize as _papyrus_standardize,
+    )
+
+
+except ImportError as e:
+    PAPYRUS_IMPORT_ERROR = e
+
+    def _papyrus_standardize(mol, **kwargs):
+        """placeholder for papyrus_standardize in case imports dont work.
+        raises respective import error when called"""
+        lg.warning("""
+            please install `papyrus_structure_pipeline` if you want to use
+            Papyrus standardisation
+            """)
+        raise PAPYRUS_IMPORT_ERROR