fairfetched 0.0.1.dev202607271__tar.gz

fairfetched-0.0.1.dev202607271/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 lucina-may nollen
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

fairfetched-0.0.1.dev202607271/PKG-INFO

@@ -0,0 +1,153 @@
+Metadata-Version: 2.4
+Name: fairfetched
+Version: 0.0.1.dev202607271
+Summary: data APIs for reproducible data fetching in cheminformatics
+Project-URL: Homepage, https://github.com/lucinamay/fairfetched
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: polars>=1.38.1
+Provides-Extra: track
+Requires-Dist: rich>=14.3.3; extra == "track"
+Provides-Extra: rdkit
+Requires-Dist: rdkit>=2025.9.6; extra == "rdkit"
+Provides-Extra: standardise
+Requires-Dist: fairfetched[rdkit]; extra == "standardise"
+Requires-Dist: chembl-structure-pipeline>=1.2.4; extra == "standardise"
+Requires-Dist: papyrus-structure-pipeline>=0.0.5; extra == "standardise"
+Provides-Extra: dev
+Requires-Dist: pytest>=9.0.2; extra == "dev"
+Requires-Dist: sqlalchemy>=2.0.46; extra == "dev"
+Requires-Dist: fairfetched[standardise]; extra == "dev"
+Requires-Dist: rust-just>=1.46.0; extra == "dev"
+Dynamic: license-file
+
+# fairfetched
+
+data APIs for reproducible data fetching in cheminformatics in line with FAIR principles.
+the tool is designed such that all raw data is downloaded and kept in one central place (i.e. CHEMBL 37 as the original .db files), and the user has a fast, lightweight, intuitive API to filter / query the data as required for any particular project.
+
+# installation
+
+you can install this package through
+`uv add fairfetched` (recommended)
+
+or if you do not use the uv package manager:
+`pip install fairfetched`
+
+# examples
+
+you can download Chembl or Papyrus through:
+
+```python
+from fairfetched.get import Chembl, Papyrus
+mychembl = Chembl.from_latest() # this downloads Chembl raw files + extracts parquet files to wherever you
+                                # have set the environment variable FAIRFETCHED_HOME, PYSTOW_HOME,
+                                # or <HOME>/.data if not in environment variables.
+                                # from there, fairfetched saves it to a folder chembl/<version>
+
+mychembl.lfs                  # a dictionary of all chembl files in polars LazyFrame format, scanned directly from the extracted .parquet files
+
+
+mychembl.parquet_paths   # the paths to the parquet-converted tabular data files in the Chembl .db file
+
+mychembl.raw_paths            # the paths to the raw chembl file as downloaded from Chembl. currently does include an uncompressed .db file
+
+mychembl.compounds            # NOT YET IMPLEMENTED !! convenience alias for mychembl.compose()["compounds"], which uses mychembl.lfs LazyFrame joins to obtain an intuitive join of the data.
+                              # from there, you can
+```
+
+### examples of how to use the LazyFrames:
+
+#### checking which columns+datatypes are in the file, so that you can choose to join them:
+
+```python
+>>> mychembl.lfs["activities"].collect_schema()
+Schema({'activity_id': Int64, 'assay_id': Int64, 'doc_id': Int64, 'record_id': Int64, 'molregno': Int64, 'standard_relation': String, 'standard_value': Float64, 'standard_units': String, 'standard_flag': Int64, 'standard_type': String, 'activity_comment': String, 'data_validity_comment': String, 'potential_duplicate': Int64, 'pchembl_value': Float64, 'bao_endpoint': String, 'uo_units': String, 'qudt_units': String, 'toid': Int64, 'upper_value': Float64, 'standard_upper_value': Null, 'src_id': Int64, 'type': String, 'relation': String, 'value': Float64, 'units': String, 'text_value': String, 'standard_text_value': String, 'action_type': String})
+```
+
+#### selecting all entries based on doc_id:
+
+```python
+>>> mychembl.lfs["activities"].filter(doc_id=89530).drop_nulls("units").collect()
+shape: (107, 28)
+┌─────────────┬──────────┬────────┬───────────┬───┬───────┬────────────┬─────────────────────┬─────────────┐
+│ activity_id ┆ assay_id ┆ doc_id ┆ record_id ┆ … ┆ units ┆ text_value ┆ standard_text_value ┆ action_type │
+│ ---         ┆ ---      ┆ ---    ┆ ---       ┆   ┆ ---   ┆ ---        ┆ ---                 ┆ ---         │
+│ i64         ┆ i64      ┆ i64    ┆ i64       ┆   ┆ str   ┆ str        ┆ str                 ┆ str         │
+╞═════════════╪══════════╪════════╪═══════════╪═══╪═══════╪════════════╪═════════════════════╪═════════════╡
+│ 15120638    ┆ 1431503  ┆ 89530  ┆ 2256150   ┆ … ┆ uM    ┆ null       ┆ null                ┆ null        │
+│ 15120639    ┆ 1431503  ┆ 89530  ┆ 2256151   ┆ … ┆ uM    ┆ null       ┆ null                ┆ null        │
+│ 15120640    ┆ 1431503  ┆ 89530  ┆ 2256152   ┆ … ┆ uM    ┆ null       ┆ null                ┆ null        │
+│ 15120641    ┆ 1431503  ┆ 89530  ┆ 2256153   ┆ … ┆ uM    ┆ null       ┆ null                ┆ null        │
+│ 15120642    ┆ 1431503  ┆ 89530  ┆ 2256154   ┆ … ┆ uM    ┆ null       ┆ null                ┆ null        │
+│ …           ┆ …        ┆ …      ┆ …         ┆ … ┆ …     ┆ …          ┆ …                   ┆ …           │
+│ 15125200    ┆ 1431507  ┆ 89530  ┆ 2256167   ┆ … ┆ uM    ┆ null       ┆ null                ┆ null        │
+│ 15125201    ┆ 1431507  ┆ 89530  ┆ 2256168   ┆ … ┆ uM    ┆ null       ┆ null                ┆ null        │
+│ 15125202    ┆ 1431507  ┆ 89530  ┆ 2256169   ┆ … ┆ uM    ┆ null       ┆ null                ┆ null        │
+│ 15125203    ┆ 1431507  ┆ 89530  ┆ 2256170   ┆ … ┆ uM    ┆ null       ┆ null                ┆ null        │
+│ 15125204    ┆ 1431507  ┆ 89530  ┆ 2256171   ┆ … ┆ uM    ┆ null       ┆ null                ┆ null        │
+└─────────────┴──────────┴────────┴───────────┴───┴───────┴────────────┴─────────────────────┴─────────────┘
+```
+
+#### adding compound structure info to the activities on molregno
+
+```python
+>>> mychembl.lfs["activities"].join(mychembl.lfs["compound_structures"],on="molregno",how="left",validate="m:1").head().collect()
+shape: (5, 32)
+┌─────────────┬──────────┬────────┬───────────┬───┬────────────────────────┬─────────────────────────────────┬─────────────────────────────┬─────────────────────────────────┐
+│ activity_id ┆ assay_id ┆ doc_id ┆ record_id ┆ … ┆ molfile                ┆ standard_inchi                  ┆ standard_inchi_key          ┆ canonical_smiles                │
+│ ---         ┆ ---      ┆ ---    ┆ ---       ┆   ┆ ---                    ┆ ---                             ┆ ---                         ┆ ---                             │
+│ i64         ┆ i64      ┆ i64    ┆ i64       ┆   ┆ str                    ┆ str                             ┆ str                         ┆ str                             │
+╞═════════════╪══════════╪════════╪═══════════╪═══╪════════════════════════╪═════════════════════════════════╪═════════════════════════════╪═════════════════════════════════╡
+│ 31863       ┆ 54505    ┆ 6424   ┆ 206172    ┆ … ┆                        ┆ InChI=1S/C20H12N2O2/c1-2-7-13(… ┆ BEBACPIIZGRKGG-UHFFFAOYSA-N ┆ c1ccc(-c2nc3c(-c4nc5ccccc5o4)c… │
+│             ┆          ┆        ┆           ┆   ┆      RDKit          2D ┆                                 ┆                             ┆                                 │
+│             ┆          ┆        ┆           ┆   ┆                        ┆                                 ┆                             ┆                                 │
+│             ┆          ┆        ┆           ┆   ┆  24 2…                 ┆                                 ┆                             ┆                                 │
+│ 31864       ┆ 83907    ┆ 6432   ┆ 208970    ┆ … ┆                        ┆ InChI=1S/C23H14N2O5/c1-12-5-8-… ┆ SUKVIELCKKEBOJ-UHFFFAOYSA-N ┆ Cc1ccc2oc(-c3cccc(N4C(=O)c5ccc… │
+│             ┆          ┆        ┆           ┆   ┆      RDKit          2D ┆                                 ┆                             ┆                                 │
+│             ┆          ┆        ┆           ┆   ┆                        ┆                                 ┆                             ┆                                 │
+│             ┆          ┆        ┆           ┆   ┆  30 3…                 ┆                                 ┆                             ┆                                 │
+│ 31865       ┆ 88152    ┆ 6432   ┆ 208970    ┆ … ┆                        ┆ InChI=1S/C23H14N2O5/c1-12-5-8-… ┆ SUKVIELCKKEBOJ-UHFFFAOYSA-N ┆ Cc1ccc2oc(-c3cccc(N4C(=O)c5ccc… │
+│             ┆          ┆        ┆           ┆   ┆      RDKit          2D ┆                                 ┆                             ┆                                 │
+│             ┆          ┆        ┆           ┆   ┆                        ┆                                 ┆                             ┆                                 │
+│             ┆          ┆        ┆           ┆   ┆  30 3…                 ┆                                 ┆                             ┆                                 │
+│ 31866       ┆ 83907    ┆ 6432   ┆ 208987    ┆ … ┆                        ┆ InChI=1S/C30H20N2O7/c1-37-24-6… ┆ ZFJHZUAZBGPPQK-UHFFFAOYSA-N ┆ COc1ccccc1-c1ccc2oc(-c3ccc(OC)… │
+│             ┆          ┆        ┆           ┆   ┆      RDKit          2D ┆                                 ┆                             ┆                                 │
+│             ┆          ┆        ┆           ┆   ┆                        ┆                                 ┆                             ┆                                 │
+│             ┆          ┆        ┆           ┆   ┆  39 4…                 ┆                                 ┆                             ┆                                 │
+│ 31867       ┆ 88153    ┆ 6432   ┆ 208987    ┆ … ┆                        ┆ InChI=1S/C30H20N2O7/c1-37-24-6… ┆ ZFJHZUAZBGPPQK-UHFFFAOYSA-N ┆ COc1ccccc1-c1ccc2oc(-c3ccc(OC)… │
+│             ┆          ┆        ┆           ┆   ┆      RDKit          2D ┆                                 ┆                             ┆                                 │
+│             ┆          ┆        ┆           ┆   ┆                        ┆                                 ┆                             ┆                                 │
+│             ┆          ┆        ┆           ┆   ┆  39 4…                 ┆                                 ┆                             ┆                                 │
+└─────────────┴──────────┴────────┴───────────┴───┴────────────────────────┴─────────────────────────────────┴─────────────────────────────┴─────────────────────────────────┘
+```
+
+#### move it to pandas for direct drop-in use (if you really want pandas...)
+
+ideally as far down the line after you complete all filtering, you call `.collect().to_pandas()` (see polars documentation for more info)
+
+```
+mychembl.lfs["activities"].collect().to_pandas()
+```
+
+# roadmap
+
+- [ ] papyrus database support
+    - [x] papyrus latest version download
+    - [x] simple nested filtering
+    - [ ] efficient nested filtering
+    - [ ] all-version support
+    - [ ] built-in pivots
+- [ ] chembl database support
+    - [x] database to tables (parquet)
+    - [ ] intuitive pre-merged flat files
+    - [ ] database visualisation
+    - [ ] remove the need for storing uncompressed .db
+- [ ] reproducion from downloaded raw file
+- [ ] reproducible molecular (and protein?) standardisation
+- [ ] automated time-url logging and manifest files
+- [ ] well-organised logging
+- [ ] dependency minimisation
+- [ ] other database support
+- [ ] preservation of api and parsing logic per major version

fairfetched-0.0.1.dev202607271/README.md

@@ -0,0 +1,129 @@
+# fairfetched
+
+data APIs for reproducible data fetching in cheminformatics in line with FAIR principles.
+the tool is designed such that all raw data is downloaded and kept in one central place (i.e. CHEMBL 37 as the original .db files), and the user has a fast, lightweight, intuitive API to filter / query the data as required for any particular project.
+
+# installation
+
+you can install this package through
+`uv add fairfetched` (recommended)
+
+or if you do not use the uv package manager:
+`pip install fairfetched`
+
+# examples
+
+you can download Chembl or Papyrus through:
+
+```python
+from fairfetched.get import Chembl, Papyrus
+mychembl = Chembl.from_latest() # this downloads Chembl raw files + extracts parquet files to wherever you
+                                # have set the environment variable FAIRFETCHED_HOME, PYSTOW_HOME,
+                                # or <HOME>/.data if not in environment variables.
+                                # from there, fairfetched saves it to a folder chembl/<version>
+
+mychembl.lfs                  # a dictionary of all chembl files in polars LazyFrame format, scanned directly from the extracted .parquet files
+
+
+mychembl.parquet_paths   # the paths to the parquet-converted tabular data files in the Chembl .db file
+
+mychembl.raw_paths            # the paths to the raw chembl file as downloaded from Chembl. currently does include an uncompressed .db file
+
+mychembl.compounds            # NOT YET IMPLEMENTED !! convenience alias for mychembl.compose()["compounds"], which uses mychembl.lfs LazyFrame joins to obtain an intuitive join of the data.
+                              # from there, you can
+```
+
+### examples of how to use the LazyFrames:
+
+#### checking which columns+datatypes are in the file, so that you can choose to join them:
+
+```python
+>>> mychembl.lfs["activities"].collect_schema()
+Schema({'activity_id': Int64, 'assay_id': Int64, 'doc_id': Int64, 'record_id': Int64, 'molregno': Int64, 'standard_relation': String, 'standard_value': Float64, 'standard_units': String, 'standard_flag': Int64, 'standard_type': String, 'activity_comment': String, 'data_validity_comment': String, 'potential_duplicate': Int64, 'pchembl_value': Float64, 'bao_endpoint': String, 'uo_units': String, 'qudt_units': String, 'toid': Int64, 'upper_value': Float64, 'standard_upper_value': Null, 'src_id': Int64, 'type': String, 'relation': String, 'value': Float64, 'units': String, 'text_value': String, 'standard_text_value': String, 'action_type': String})
+```
+
+#### selecting all entries based on doc_id:
+
+```python
+>>> mychembl.lfs["activities"].filter(doc_id=89530).drop_nulls("units").collect()
+shape: (107, 28)
+┌─────────────┬──────────┬────────┬───────────┬───┬───────┬────────────┬─────────────────────┬─────────────┐
+│ activity_id ┆ assay_id ┆ doc_id ┆ record_id ┆ … ┆ units ┆ text_value ┆ standard_text_value ┆ action_type │
+│ ---         ┆ ---      ┆ ---    ┆ ---       ┆   ┆ ---   ┆ ---        ┆ ---                 ┆ ---         │
+│ i64         ┆ i64      ┆ i64    ┆ i64       ┆   ┆ str   ┆ str        ┆ str                 ┆ str         │
+╞═════════════╪══════════╪════════╪═══════════╪═══╪═══════╪════════════╪═════════════════════╪═════════════╡
+│ 15120638    ┆ 1431503  ┆ 89530  ┆ 2256150   ┆ … ┆ uM    ┆ null       ┆ null                ┆ null        │
+│ 15120639    ┆ 1431503  ┆ 89530  ┆ 2256151   ┆ … ┆ uM    ┆ null       ┆ null                ┆ null        │
+│ 15120640    ┆ 1431503  ┆ 89530  ┆ 2256152   ┆ … ┆ uM    ┆ null       ┆ null                ┆ null        │
+│ 15120641    ┆ 1431503  ┆ 89530  ┆ 2256153   ┆ … ┆ uM    ┆ null       ┆ null                ┆ null        │
+│ 15120642    ┆ 1431503  ┆ 89530  ┆ 2256154   ┆ … ┆ uM    ┆ null       ┆ null                ┆ null        │
+│ …           ┆ …        ┆ …      ┆ …         ┆ … ┆ …     ┆ …          ┆ …                   ┆ …           │
+│ 15125200    ┆ 1431507  ┆ 89530  ┆ 2256167   ┆ … ┆ uM    ┆ null       ┆ null                ┆ null        │
+│ 15125201    ┆ 1431507  ┆ 89530  ┆ 2256168   ┆ … ┆ uM    ┆ null       ┆ null                ┆ null        │
+│ 15125202    ┆ 1431507  ┆ 89530  ┆ 2256169   ┆ … ┆ uM    ┆ null       ┆ null                ┆ null        │
+│ 15125203    ┆ 1431507  ┆ 89530  ┆ 2256170   ┆ … ┆ uM    ┆ null       ┆ null                ┆ null        │
+│ 15125204    ┆ 1431507  ┆ 89530  ┆ 2256171   ┆ … ┆ uM    ┆ null       ┆ null                ┆ null        │
+└─────────────┴──────────┴────────┴───────────┴───┴───────┴────────────┴─────────────────────┴─────────────┘
+```
+
+#### adding compound structure info to the activities on molregno
+
+```python
+>>> mychembl.lfs["activities"].join(mychembl.lfs["compound_structures"],on="molregno",how="left",validate="m:1").head().collect()
+shape: (5, 32)
+┌─────────────┬──────────┬────────┬───────────┬───┬────────────────────────┬─────────────────────────────────┬─────────────────────────────┬─────────────────────────────────┐
+│ activity_id ┆ assay_id ┆ doc_id ┆ record_id ┆ … ┆ molfile                ┆ standard_inchi                  ┆ standard_inchi_key          ┆ canonical_smiles                │
+│ ---         ┆ ---      ┆ ---    ┆ ---       ┆   ┆ ---                    ┆ ---                             ┆ ---                         ┆ ---                             │
+│ i64         ┆ i64      ┆ i64    ┆ i64       ┆   ┆ str                    ┆ str                             ┆ str                         ┆ str                             │
+╞═════════════╪══════════╪════════╪═══════════╪═══╪════════════════════════╪═════════════════════════════════╪═════════════════════════════╪═════════════════════════════════╡
+│ 31863       ┆ 54505    ┆ 6424   ┆ 206172    ┆ … ┆                        ┆ InChI=1S/C20H12N2O2/c1-2-7-13(… ┆ BEBACPIIZGRKGG-UHFFFAOYSA-N ┆ c1ccc(-c2nc3c(-c4nc5ccccc5o4)c… │
+│             ┆          ┆        ┆           ┆   ┆      RDKit          2D ┆                                 ┆                             ┆                                 │
+│             ┆          ┆        ┆           ┆   ┆                        ┆                                 ┆                             ┆                                 │
+│             ┆          ┆        ┆           ┆   ┆  24 2…                 ┆                                 ┆                             ┆                                 │
+│ 31864       ┆ 83907    ┆ 6432   ┆ 208970    ┆ … ┆                        ┆ InChI=1S/C23H14N2O5/c1-12-5-8-… ┆ SUKVIELCKKEBOJ-UHFFFAOYSA-N ┆ Cc1ccc2oc(-c3cccc(N4C(=O)c5ccc… │
+│             ┆          ┆        ┆           ┆   ┆      RDKit          2D ┆                                 ┆                             ┆                                 │
+│             ┆          ┆        ┆           ┆   ┆                        ┆                                 ┆                             ┆                                 │
+│             ┆          ┆        ┆           ┆   ┆  30 3…                 ┆                                 ┆                             ┆                                 │
+│ 31865       ┆ 88152    ┆ 6432   ┆ 208970    ┆ … ┆                        ┆ InChI=1S/C23H14N2O5/c1-12-5-8-… ┆ SUKVIELCKKEBOJ-UHFFFAOYSA-N ┆ Cc1ccc2oc(-c3cccc(N4C(=O)c5ccc… │
+│             ┆          ┆        ┆           ┆   ┆      RDKit          2D ┆                                 ┆                             ┆                                 │
+│             ┆          ┆        ┆           ┆   ┆                        ┆                                 ┆                             ┆                                 │
+│             ┆          ┆        ┆           ┆   ┆  30 3…                 ┆                                 ┆                             ┆                                 │
+│ 31866       ┆ 83907    ┆ 6432   ┆ 208987    ┆ … ┆                        ┆ InChI=1S/C30H20N2O7/c1-37-24-6… ┆ ZFJHZUAZBGPPQK-UHFFFAOYSA-N ┆ COc1ccccc1-c1ccc2oc(-c3ccc(OC)… │
+│             ┆          ┆        ┆           ┆   ┆      RDKit          2D ┆                                 ┆                             ┆                                 │
+│             ┆          ┆        ┆           ┆   ┆                        ┆                                 ┆                             ┆                                 │
+│             ┆          ┆        ┆           ┆   ┆  39 4…                 ┆                                 ┆                             ┆                                 │
+│ 31867       ┆ 88153    ┆ 6432   ┆ 208987    ┆ … ┆                        ┆ InChI=1S/C30H20N2O7/c1-37-24-6… ┆ ZFJHZUAZBGPPQK-UHFFFAOYSA-N ┆ COc1ccccc1-c1ccc2oc(-c3ccc(OC)… │
+│             ┆          ┆        ┆           ┆   ┆      RDKit          2D ┆                                 ┆                             ┆                                 │
+│             ┆          ┆        ┆           ┆   ┆                        ┆                                 ┆                             ┆                                 │
+│             ┆          ┆        ┆           ┆   ┆  39 4…                 ┆                                 ┆                             ┆                                 │
+└─────────────┴──────────┴────────┴───────────┴───┴────────────────────────┴─────────────────────────────────┴─────────────────────────────┴─────────────────────────────────┘
+```
+
+#### move it to pandas for direct drop-in use (if you really want pandas...)
+
+ideally as far down the line after you complete all filtering, you call `.collect().to_pandas()` (see polars documentation for more info)
+
+```
+mychembl.lfs["activities"].collect().to_pandas()
+```
+
+# roadmap
+
+- [ ] papyrus database support
+    - [x] papyrus latest version download
+    - [x] simple nested filtering
+    - [ ] efficient nested filtering
+    - [ ] all-version support
+    - [ ] built-in pivots
+- [ ] chembl database support
+    - [x] database to tables (parquet)
+    - [ ] intuitive pre-merged flat files
+    - [ ] database visualisation
+    - [ ] remove the need for storing uncompressed .db
+- [ ] reproducion from downloaded raw file
+- [ ] reproducible molecular (and protein?) standardisation
+- [ ] automated time-url logging and manifest files
+- [ ] well-organised logging
+- [ ] dependency minimisation
+- [ ] other database support
+- [ ] preservation of api and parsing logic per major version

fairfetched-0.0.1.dev202607271/fairfetched/get/__init__.py

@@ -0,0 +1,9 @@
+from .dataset import Chembl, Papyrus
+
+__all__ = ["Chembl", "Papyrus"]
+
+if __name__ == "__main__":
+    from .papyrus import ensure_raw_files, latest
+
+    print(latest())
+    ensure_raw_files(latest())

fairfetched-0.0.1.dev202607271/fairfetched/get/chembl.py

@@ -0,0 +1,219 @@
+from collections.abc import Sequence
+from pathlib import Path
+from typing import Any
+
+import polars as pl
+
+from fairfetched.utils import (
+    BASE_DIR,
+    ensure_sqlite_db_to_parquets,
+    ensure_untarred_sqlite,
+    ensure_url,
+    file_suffix_from_url,
+    lowercase_columns,
+)
+from fairfetched.utils.typing import BioactivityDBViews
+
+CHEMBL_DIR = BASE_DIR / "chembl"
+
+
+def _format_version(version: float | str) -> str:
+    if isinstance(version, int | float):
+        version = str(version)
+    if isinstance(version, Sequence) and not isinstance(version, str):
+        raise TypeError(f"invalid version type: {type(version)}")
+    if not isinstance(version, str):
+        try:
+            version = str(version)
+        except Exception:
+            raise TypeError(f"invalid version type: {type(version)}")
+
+    version = version.lstrip("0")
+    if "." in version:
+        version = version.split(".")[0].zfill(2) + "." + version.split(".")[1]
+    # for canonicalize the version number 22.1 and 24.1 and left pad with a zero if needed
+    return version.replace(".", "_").replace("_0", "").zfill(2)
+
+
+def _version_to_url(version: str):
+    base = "https://ftp.ebi.ac.uk/pub/databases/chembl/ChEMBLdb/releases"
+    return f"{base}/chembl_{version}/chembl_{version}_sqlite.tar.gz"
+
+
+CHEMBL_VERSIONS: dict[str, dict[str, str]] = {
+    version: {"sql_db": _version_to_url(version)}
+    for version in sorted(map(_format_version, list(range(1, 38)) + ["24_1", "22_1"]))
+}
+
+
+def available_versions() -> tuple[str, ...]:
+    return tuple(CHEMBL_VERSIONS.keys())
+
+
+def latest() -> str:
+    return available_versions()[-1]
+
+
+def source_urls(version: str) -> dict[str, str]:
+    return CHEMBL_VERSIONS[str(version)]
+
+
+def ensure_raw_files(
+    version: str, raw_dir: Path | str | None = None, force=False
+) -> dict[str, Path]:
+    """Download the original SQL database with its original name and compression."""
+    if raw_dir is None:
+        raw_dir = CHEMBL_DIR / version
+    raw_dir = Path(raw_dir)
+    return {
+        name: ensure_url(
+            url=url, path=raw_dir / f"{name}{file_suffix_from_url(url)}", force=force
+        )
+        for name, url in source_urls(version).items()
+    }
+
+
+def ensure_parquet_tables(
+    raw_paths: dict[str, Path], table_dir: Path | str | Any | None = None
+) -> dict[str, Path]:
+    sql_tar_gz_path = raw_paths["sql_db"]
+    if table_dir is None:
+        table_dir = Path(sql_tar_gz_path).parent / "extracted"
+    table_dir = Path(table_dir)
+    table_dir.mkdir(exist_ok=True, parents=True)
+
+    raw_sql = ensure_untarred_sqlite(sql_tar_gz_path)
+    # the untarred should also stay so that we have access.....
+    # #@TODO: perhaps make tables deterministic for chembl to circumvent
+    parquets = ensure_sqlite_db_to_parquets(raw_sql, cache_dir=table_dir, force=False)
+
+    return parquets
+
+
+def cleanly_scan_parquet(path_: Path | str) -> pl.LazyFrame:
+    """scans parquet paths and lazily handles null value conversion to None"""
+    return (
+        pl.scan_parquet(path_)
+        .pipe(lowercase_columns)
+        .fill_nan(None)
+        .with_columns(
+            pl.col(pl.String).replace({"": None}),
+        )
+    )
+
+
+def cleanly_scan_parquet_tables(
+    parquet_paths: dict[str, Path],
+) -> dict[str, pl.LazyFrame]:
+    """scans parquet paths and lazily handles null value conversion to None"""
+    return {name: cleanly_scan_parquet(path_) for name, path_ in parquet_paths.items()}
+
+
+# def build_views(lfs: dict[str, pl.LazyFrame]) -> ComposedLFDict:
+def build_views(parquet_paths: dict[str, Path]) -> BioactivityDBViews:
+    """Build joined domain views from the scanned source tables."""
+    return {
+        "bioactivity": _bioactivities(parquet_paths),
+        "compounds": _compounds(parquet_paths),
+        "proteins": cleanly_scan_parquet(parquet_paths["protein"]),
+        "components": _components(parquet_paths),
+    }
+
+
+def _bioactivities(parquet_paths: dict[str, Path]) -> pl.LazyFrame:
+    lfs = cleanly_scan_parquet_tables(parquet_paths)
+    return (
+        # dfs["activity_properties"]
+        # .join(dfs["activities"], on="activity_id", how="left")
+        # .join(dfs["action_type"], on="action_type", how="left")
+        # .join
+        lfs["bioactivity"]
+        .join(
+            lfs["protein"],
+            on="target_id",
+            how="left",
+            maintain_order="left",
+            validate="m:1",  # one unique protein only from right, can reoccur within compounds.
+        )
+        .join(lfs["action_type"], on="action_type", how="left", suffix="_action_type")
+        .join(
+            lfs["assays"], on="assay_id", how="left", suffix="_assay"
+        )  # doc_ids to this
+        .join(lfs["assay_type"], on="assay_type", how="left", suffix="_assay_type")
+        # .join(lfs, on="bao_format")
+        # .join(dfs[])
+    )
+
+
+def _components(parquet_paths: dict[str, Path]) -> pl.LazyFrame:
+    lfs = cleanly_scan_parquet_tables(parquet_paths)
+    return (
+        lfs["component_sequences"]
+        .join(
+            lfs["component_class"],
+            on="component_id",
+            how="left",
+            suffix="_class",
+            validate="1:m",
+        )
+        .join(
+            lfs["component_domains"].join(
+                lfs["domains"],
+                on="domain_id",
+                how="left",
+                suffix="_domains",
+                validate="m:1",
+            ),
+            on="component_id",
+            how="left",
+            suffix="_domains",
+            validate="1:m",
+        )
+        # .join(
+        #     dfs["component_synonyms"],
+        #     on="component_id",
+        #     how="left",
+        #     suffix="_synonyms",
+        #     validate="m:m",
+        # )
+    )
+
+
+def _compounds(parquet_paths: dict[str, Path]) -> pl.LazyFrame:
+    lfs = cleanly_scan_parquet_tables(parquet_paths)
+    return (
+        lfs["molecule_dictionary"]
+        .join(
+            lfs["compound_properties"],
+            on="molregno",
+            how="left",
+            suffix="_compound_properties",
+            validate="1:1",
+        )
+        .join(
+            lfs["compound_structures"],
+            on="molregno",
+            how="left",
+            suffix="_compound_structures",
+            validate="1:1",
+        )
+        .join(
+            lfs["compound_records"].join(
+                lfs["docs"], on="doc_id", how="left", suffix="_doc", validate="m:1"
+            ),
+            on="molregno",
+            how="left",
+            suffix="_compound_records",
+            validate="1:m",
+        )
+        .join(
+            lfs["compound_structural_alerts"],
+            #     .join(
+            #     dfs["docs"], on="doc_id", how="left", suffix="_doc",validate="m:1"
+            # )
+            on="molregno",
+            how="left",
+            suffix="_compound_structural_alerts",
+            validate="1:m",
+        )
+    )