cellmaps-utils 0.8.0__tar.gz → 0.9.0__tar.gz

{cellmaps_utils-0.8.0 → cellmaps_utils-0.9.0}/HISTORY.rst

@@ -2,6 +2,13 @@
 History
 =======
 
+0.9.0 (2025-07-03)
+-------------------
+
+* Add statistics generator for ro-crates
+
+* Fix provenance bugs for new versions of ro-crate
+
 0.8.0 (2025-04-23)
 -------------------
 

{cellmaps_utils-0.8.0 → cellmaps_utils-0.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.1
 Name: cellmaps_utils
-Version: 0.8.0
+Version: 0.9.0
 Summary: Contains utilities needed by Cell Maps tools
 Home-page: https://github.com/idekerlab/cellmaps_utils
 Author: Cell Maps team
@@ -27,17 +27,6 @@ Requires-Dist: numpy<2.0.0,>=1.24.4
 Requires-Dist: dill<0.4.0,>=0.3.8
 Requires-Dist: tqdm<5.0.0,>=4.66.6
 Requires-Dist: requests<3.0.0,>=2.31.0
-Dynamic: author
-Dynamic: author-email
-Dynamic: classifier
-Dynamic: description
-Dynamic: description-content-type
-Dynamic: home-page
-Dynamic: keywords
-Dynamic: license
-Dynamic: license-file
-Dynamic: requires-dist
-Dynamic: summary
 
 =============================
 Cell Maps Pipeline Utilities
@@ -120,7 +109,7 @@ Output:
    dist                 builds source and wheel package
    install              install the package to the active Python's site-packages
 
-Before running tests, please install ``pip install -r requirements_dev``
+Before running tests, please install ``pip install -r requirements_dev.txt``
 
 For developers
 -------------------------------------------
@@ -160,6 +149,13 @@ This package was created with Cookiecutter_ and the `audreyr/cookiecutter-pypack
 History
 =======
 
+0.9.0 (2025-07-03)
+-------------------
+
+* Add statistics generator for ro-crates
+
+* Fix provenance bugs for new versions of ro-crate
+
 0.8.0 (2025-04-23)
 -------------------
 

{cellmaps_utils-0.8.0 → cellmaps_utils-0.9.0}/README.rst

@@ -79,7 +79,7 @@ Output:
    dist                 builds source and wheel package
    install              install the package to the active Python's site-packages
 
-Before running tests, please install ``pip install -r requirements_dev``
+Before running tests, please install ``pip install -r requirements_dev.txt``
 
 For developers
 -------------------------------------------

{cellmaps_utils-0.8.0 → cellmaps_utils-0.9.0}/cellmaps_utils/__init__.py

@@ -4,6 +4,6 @@
 
 __author__ = 'Cell Maps team'
 __email__ = 'tools@cm4ai.org'
-__version__ = '0.8.0'
+__version__ = '0.9.0'
 __repo_url__ = 'https://github.com/idekerlab/cellmaps_utils'
 __description__ = 'Contains utilities needed by Cell Maps tools'

{cellmaps_utils-0.8.0 → cellmaps_utils-0.9.0}/cellmaps_utils/provenance.py

@@ -361,7 +361,13 @@ class ProvenanceUtil(object):
             data = rocrate
         else:
             data = self.get_rocrate_as_dict(rocrate)
-        return data['@id']
+        if '@id' not in data.keys():
+            rocrate_id = data.get('@graph')[1].get('@id', None)
+            if rocrate_id is None:
+                raise CellMapsProvenanceError('ID of ro-crate not found')
+            return rocrate_id
+        else:
+            return data['@id']
 
     def get_name_project_org_of_rocrate(self, rocrate):
         """
@@ -395,6 +401,9 @@ class ProvenanceUtil(object):
         else:
             data = self.get_rocrate_as_dict(rocrate)
 
+        if 'name' not in data.keys():
+            data = data.get('@graph')[1]
+
         name = data['name']
         description = data['description']
         keywords = data['keywords']

cellmaps_utils-0.9.0/cellmaps_utils/statsgenerator.py

@@ -0,0 +1,237 @@
+import os
+import argparse
+import json
+import pandas as pd
+import re
+from collections import defaultdict
+from functools import lru_cache
+import mygene
+
+CELL_LINES = ['MDA-MB-468', 'Kolf2.1J']
+
+
+@lru_cache(maxsize=10000)
+def query_gene_symbols(ensembl_id_tuple):
+    mg = mygene.MyGeneInfo()
+    results = mg.querymany(list(ensembl_id_tuple), scopes='ensembl.gene', fields='symbol', species='human')
+    mapping = {}
+    for entry in results:
+        if not entry.get('notfound') and 'symbol' in entry:
+            mapping[entry['query']] = entry['symbol']
+        else:
+            mapping[entry['query']] = f"UNKNOWN:{entry['query']}"
+    return mapping
+
+
+def get_gene_symbols_from_ensembl_ids(ensembl_ids):
+    unique_ids = set(ensembl_ids)
+    return query_gene_symbols(tuple(unique_ids))
+
+
+def extract_cell_line_names(json_path):
+    known_cell_lines = CELL_LINES
+    found = []
+    try:
+        with open(json_path, "r", encoding="utf-8") as f:
+            data = json.load(f)
+
+        def collect_texts(d):
+            texts = []
+            for value in d.values():
+                if isinstance(value, str):
+                    texts.append(value)
+                elif isinstance(value, list):
+                    texts.extend([v for v in value if isinstance(v, str)])
+                elif isinstance(value, dict):
+                    texts.extend(collect_texts(value))
+            return texts
+
+        all_text = collect_texts(data)
+        if "@graph" in data and isinstance(data["@graph"], list):
+            for entry in data["@graph"]:
+                if isinstance(entry, dict):
+                    all_text.extend(collect_texts(entry))
+        combined_text = " ".join(all_text)
+
+        for cl in known_cell_lines:
+            if re.search(rf'\b{re.escape(cl)}\b', combined_text, re.IGNORECASE):
+                found.append(cl)
+
+        return sorted(found) if found else ["<not found>"]
+    except Exception as e:
+        return [f"Error reading file: {e}"]
+
+
+def find_first_csv_or_tsv(folder):
+    for fname in os.listdir(folder):
+        if fname.lower().endswith(".csv") or fname.lower().endswith(".tsv"):
+            return os.path.join(folder, fname)
+    return None
+
+
+def get_ensembl_ids(file_path):
+    try:
+        if file_path.endswith(".tsv"):
+            df = pd.read_csv(file_path, sep='\t')
+        else:
+            df = pd.read_csv(file_path)
+        if "ENSEMBL ID" in df.columns:
+            return df["ENSEMBL ID"].dropna().tolist()
+    except Exception:
+        pass
+    return []
+
+
+def get_protein_names(file_path):
+    try:
+        if file_path.endswith(".tsv"):
+            df = pd.read_csv(file_path, sep='\t')
+        else:
+            df = pd.read_csv(file_path)
+        if "Bait" in df.columns and "PreyGene.x" in df.columns:
+            bait = df["Bait"].dropna().tolist()
+            prey = df["PreyGene.x"].dropna().tolist()
+            all_proteins = bait + prey
+            cleaned = {p.split("_")[0] for p in all_proteins}
+            return cleaned
+    except Exception:
+        pass
+    return set()
+
+
+def count_unique_image_filenames_per_cell_line(images_crates):
+    channels = ['blue', 'green', 'red', 'yellow']
+    cell_line_to_filenames = defaultdict(set)
+
+    for crate in images_crates:
+        crate = crate.strip()
+        if not os.path.isdir(crate):
+            continue
+        metadata_path = os.path.join(crate, "ro-crate-metadata.json")
+        if not os.path.isfile(metadata_path):
+            continue
+        cell_lines = extract_cell_line_names(metadata_path)
+        for color in channels:
+            color_path = os.path.join(crate, color)
+            if not os.path.isdir(color_path):
+                continue
+            filenames = [f for f in os.listdir(color_path) if os.path.isfile(os.path.join(color_path, f))]
+            for cl in cell_lines:
+                cell_line_to_filenames[cl].update(filenames)
+
+    return {cl: len(files) for cl, files in cell_line_to_filenames.items()}
+
+
+def check_and_aggregate_images(images_crates):
+    cell_line_to_ensembl = defaultdict(set)
+
+    for path in images_crates:
+        path = path.strip()
+        metadata_path = os.path.join(path, "ro-crate-metadata.json")
+        if not os.path.isfile(metadata_path):
+            continue
+        cell_lines = extract_cell_line_names(metadata_path)
+        if not cell_lines or cell_lines == ["<not found>"]:
+            continue
+        ensembl_file = find_first_csv_or_tsv(path)
+        if not ensembl_file:
+            continue
+        ensembl_ids = get_ensembl_ids(ensembl_file)
+        if not ensembl_ids:
+            continue
+        for cell_line in cell_lines:
+            cell_line_to_ensembl[cell_line].update(ensembl_ids)
+
+    all_gene_symbols = defaultdict(set)
+    for cl, ids in cell_line_to_ensembl.items():
+        symbol_map = get_gene_symbols_from_ensembl_ids(list(ids))
+        all_gene_symbols[cl] = set(symbol_map.values())
+
+    print("\nImages Crates:")
+    for cl in CELL_LINES:
+        print(f"{cl}: {len(all_gene_symbols[cl])} unique proteins")
+    return all_gene_symbols
+
+
+def check_and_aggregate_interactions(interaction_crates):
+    cell_line_to_proteins = defaultdict(set)
+
+    for path in interaction_crates:
+        path = path.strip()
+        metadata_path = os.path.join(path, "ro-crate-metadata.json")
+        if not os.path.isfile(metadata_path):
+            continue
+        cell_lines = extract_cell_line_names(metadata_path)
+        if not cell_lines or cell_lines == ["<not found>"]:
+            continue
+        interaction_file = find_first_csv_or_tsv(path)
+        if not interaction_file:
+            continue
+        names = get_protein_names(interaction_file)
+        if not names:
+            continue
+        for cell_line in cell_lines:
+            cell_line_to_proteins[cell_line].update(names)
+
+    print("\nInteraction Crates:")
+    for cl in CELL_LINES:
+        print(f"{cl}: {len(cell_line_to_proteins[cl])} unique proteins")
+    return cell_line_to_proteins
+
+
+def save_summary_table(proteins, interactions, image_counts, output_file="statistics.csv"):
+    data = []
+    differentiation = {
+        "MDA-MB-468": ["NA"],
+        "Kolf2.1J": ["undifferentiated", "neuron", "cardiomycyte"]
+    }
+
+    for cl in CELL_LINES:
+        interaction_proteins = interactions.get(cl, set())
+        protein_crate_proteins = proteins.get(cl, set())
+        total = interaction_proteins.union(protein_crate_proteins)
+        image_count = image_counts.get(cl, "")
+
+        for diff in differentiation[cl]:
+            row = {
+                "Cell Line/Type": cl,
+                "Differentiation": diff,
+                "Protein interactions detected": len(interaction_proteins),
+                "Number of immunofluorescent images": image_count,
+                "Number immunofluorescent proteins detected": len(protein_crate_proteins),
+                "Number of genes knocked down in perturb seq": "",
+                "Total proteins investigated": len(total)
+            }
+            data.append(row)
+
+    df = pd.DataFrame(data)
+    df.to_csv(output_file, index=False)
+    print(f"\nSummary table saved to: {output_file}")
+
+
+def generate_dataset_statistics(images_crates, interaction_crates):
+    proteins = defaultdict(set)
+    interactions = defaultdict(set)
+    image_counts = defaultdict(int)
+
+    if images_crates:
+        proteins = check_and_aggregate_images(images_crates)
+        image_counts = count_unique_image_filenames_per_cell_line(images_crates)
+
+    if interaction_crates:
+        interactions = check_and_aggregate_interactions(interaction_crates)
+
+    print("\nAll:")
+    for cl in CELL_LINES:
+        union = proteins[cl] | interactions[cl]
+        print(f"{cl}: {len(union)} unique proteins")
+
+    save_summary_table(proteins, interactions, image_counts)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Generate statistics for datasets")
+    parser.add_argument("--interaction_crates", nargs='+', help="List of interaction RO-Crate folder paths.")
+    parser.add_argument("--images_crates", nargs='+', help="List of image RO-Crate folder paths.")
+    args = parser.parse_args()
+
+    generate_dataset_statistics(args.images_crates, args.interaction_crates)

{cellmaps_utils-0.8.0 → cellmaps_utils-0.9.0}/cellmaps_utils.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.1
 Name: cellmaps_utils
-Version: 0.8.0
+Version: 0.9.0
 Summary: Contains utilities needed by Cell Maps tools
 Home-page: https://github.com/idekerlab/cellmaps_utils
 Author: Cell Maps team
@@ -27,17 +27,6 @@ Requires-Dist: numpy<2.0.0,>=1.24.4
 Requires-Dist: dill<0.4.0,>=0.3.8
 Requires-Dist: tqdm<5.0.0,>=4.66.6
 Requires-Dist: requests<3.0.0,>=2.31.0
-Dynamic: author
-Dynamic: author-email
-Dynamic: classifier
-Dynamic: description
-Dynamic: description-content-type
-Dynamic: home-page
-Dynamic: keywords
-Dynamic: license
-Dynamic: license-file
-Dynamic: requires-dist
-Dynamic: summary
 
 =============================
 Cell Maps Pipeline Utilities
@@ -120,7 +109,7 @@ Output:
    dist                 builds source and wheel package
    install              install the package to the active Python's site-packages
 
-Before running tests, please install ``pip install -r requirements_dev``
+Before running tests, please install ``pip install -r requirements_dev.txt``
 
 For developers
 -------------------------------------------
@@ -160,6 +149,13 @@ This package was created with Cookiecutter_ and the `audreyr/cookiecutter-pypack
 History
 =======
 
+0.9.0 (2025-07-03)
+-------------------
+
+* Add statistics generator for ro-crates
+
+* Fix provenance bugs for new versions of ro-crate
+
 0.8.0 (2025-04-23)
 -------------------
 

{cellmaps_utils-0.8.0 → cellmaps_utils-0.9.0}/cellmaps_utils.egg-info/SOURCES.txt

@@ -27,6 +27,7 @@ cellmaps_utils/logutils.py
 cellmaps_utils/music_utils.py
 cellmaps_utils/ndexupload.py
 cellmaps_utils/provenance.py
+cellmaps_utils/statsgenerator.py
 cellmaps_utils/tabletool.py
 cellmaps_utils.egg-info/PKG-INFO
 cellmaps_utils.egg-info/SOURCES.txt
@@ -63,6 +64,7 @@ docs/versioningscheme.rst
 docs/_build/html/_static/file.png
 docs/_build/html/_static/minus.png
 docs/_build/html/_static/plus.png
+tests/.DS_Store
 tests/__init__.py
 tests/test_apmstool.py
 tests/test_basecommandlinetool.py
@@ -77,6 +79,7 @@ tests/test_logutils.py
 tests/test_ndexupload.py
 tests/test_provenance.py
 tests/test_solutiongenerator.py
+tests/test_statsgenerator.py
 tests/test_tabletool.py
 tests/test_tworeplcoelutionchallengegenerator.py
 tests/data/ddot_output.ont

{cellmaps_utils-0.8.0 → cellmaps_utils-0.9.0}/tests/test_provenance.py

@@ -753,3 +753,57 @@ class TestProvenanceUtil(unittest.TestCase):
             import time
             print(os.listdir(os.path.join(temp_dir, 'test_rocrate')))
             shutil.rmtree(temp_dir)
+
+    def test_get_rocrate_provenance_attributes_from_valid_dict(self):
+        prov = ProvenanceUtil()
+        rocrate_dict = {
+            'name': 'Test Crate',
+            'description': 'Crate for testing',
+            'keywords': ['proj', 'v1.0', 'HeLa', 'drugA', 'embedding'],
+            'isPartOf': [
+                {'@type': 'Organization', 'name': 'TestOrg'},
+                {'@type': 'Project', 'name': 'TestProject'}
+            ]
+        }
+
+        attrs = prov.get_rocrate_provenance_attributes(rocrate_dict)
+        self.assertEqual(attrs.get_name(), 'Test Crate')
+        self.assertEqual(attrs.get_description(), 'Crate for testing')
+        self.assertEqual(attrs.get_organization_name(), 'TestOrg')
+        self.assertEqual(attrs.get_project_name(), 'TestProject')
+        self.assertEqual(attrs.get_keywords(), ['proj', 'v1.0', 'HeLa', 'drugA', 'embedding'])
+
+    def test_get_rocrate_provenance_attributes_missing_fields(self):
+        prov = ProvenanceUtil()
+        rocrate_dict = {
+            'name': 'Test Crate',
+            'description': 'Missing project/org',
+            'keywords': ['x'],
+            'isPartOf': []
+        }
+
+        attrs = prov.get_rocrate_provenance_attributes(rocrate_dict)
+        self.assertEqual(attrs.get_project_name(), None)
+        self.assertEqual(attrs.get_organization_name(), None)
+
+    def test_get_id_of_rocrate_from_graph_structure(self):
+        prov = ProvenanceUtil()
+        crate_dict = {
+            '@graph': [
+                {'@id': 'ignored'},
+                {'@id': 'expected_id'}
+            ]
+        }
+        result = prov.get_id_of_rocrate(crate_dict)
+        self.assertEqual(result, 'expected_id')
+
+    def test_get_id_of_rocrate_fails_if_id_not_found(self):
+        prov = ProvenanceUtil()
+        crate_dict = {
+            '@graph': [
+                {'@id': 'ignored'},
+                {'wrongkey': 'oops'}
+            ]
+        }
+        with self.assertRaises(CellMapsProvenanceError):
+            prov.get_id_of_rocrate(crate_dict)

cellmaps_utils-0.9.0/tests/test_statsgenerator.py

@@ -0,0 +1,127 @@
+import unittest
+import tempfile
+import os
+import json
+from unittest.mock import patch
+
+import pandas as pd
+from cellmaps_utils.statsgenerator import (
+    extract_cell_line_names,
+    find_first_csv_or_tsv,
+    get_ensembl_ids,
+    count_unique_image_filenames_per_cell_line,
+    get_protein_names,
+    save_summary_table,
+    generate_dataset_statistics
+)
+
+class TestDatasetStatistics(unittest.TestCase):
+
+    def setUp(self):
+        self.test_dir = tempfile.TemporaryDirectory()
+        self.base_path = self.test_dir.name
+
+    def tearDown(self):
+        self.test_dir.cleanup()
+
+    def test_extract_cell_line_names_found(self):
+        metadata = {
+            "@graph": [{"description": "Experiment using MDA-MB-468 cell line"}]
+        }
+        file_path = os.path.join(self.base_path, "ro-crate-metadata.json")
+        with open(file_path, "w") as f:
+            json.dump(metadata, f)
+
+        result = extract_cell_line_names(file_path)
+        self.assertEqual(result, ["MDA-MB-468"])
+
+    def test_extract_cell_line_names_not_found(self):
+        metadata = {
+            "@graph": [{"description": "No matching cell line"}]
+        }
+        file_path = os.path.join(self.base_path, "ro-crate-metadata.json")
+        with open(file_path, "w") as f:
+            json.dump(metadata, f)
+
+        result = extract_cell_line_names(file_path)
+        self.assertEqual(result, ["<not found>"])
+
+    def test_find_first_csv_or_tsv(self):
+        file_path = os.path.join(self.base_path, "data.tsv")
+        with open(file_path, "w") as f:
+            f.write("ENSEMBL ID\nENSG000001234\n")
+
+        found = find_first_csv_or_tsv(self.base_path)
+        self.assertTrue(found.endswith("data.tsv"))
+
+    def test_get_ensembl_ids(self):
+        df = pd.DataFrame({"ENSEMBL ID": ["ENSG000001", "ENSG000002"]})
+        file_path = os.path.join(self.base_path, "mock.csv")
+        df.to_csv(file_path, index=False)
+
+        result = get_ensembl_ids(file_path)
+        self.assertEqual(result, ["ENSG000001", "ENSG000002"])
+
+    def test_count_unique_image_filenames_per_cell_line(self):
+        meta = {
+            "@graph": [{"name": "image crate for Kolf2.1J"}]
+        }
+        metadata_path = os.path.join(self.base_path, "ro-crate-metadata.json")
+        with open(metadata_path, "w") as f:
+            json.dump(meta, f)
+
+        # Setup: image file in "green"
+        green_path = os.path.join(self.base_path, "green")
+        os.makedirs(green_path)
+        with open(os.path.join(green_path, "img1.png"), "w") as f:
+            f.write("fake image")
+
+        result = count_unique_image_filenames_per_cell_line([self.base_path])
+        self.assertEqual(result["Kolf2.1J"], 1)
+
+    def test_get_protein_names(self):
+        df = pd.DataFrame({
+            "Bait": ["TP53_abc", "BRCA1_123"],
+            "PreyGene.x": ["EGFR_1", "MYC_test"]
+        })
+        file_path = os.path.join(self.base_path, "interact.csv")
+        df.to_csv(file_path, index=False)
+
+        result = get_protein_names(file_path)
+        self.assertEqual(result, {"TP53", "BRCA1", "EGFR", "MYC"})
+
+    def test_save_summary_table(self):
+        proteins = {"MDA-MB-468": {"A", "B"}, "Kolf2.1J": {"X", "Y"}}
+        interactions = {"MDA-MB-468": {"B", "C"}, "Kolf2.1J": {"Y"}}
+        image_counts = {"MDA-MB-468": 4, "Kolf2.1J": 10}
+
+        file_path = os.path.join(self.base_path, "output.csv")
+        save_summary_table(proteins, interactions, image_counts, output_file=file_path)
+
+        df = pd.read_csv(file_path)
+        self.assertEqual(len(df), 4)  # 1 MDA + 3 Kolf2 rows
+        self.assertIn("Total proteins investigated", df.columns)
+        self.assertEqual(df[df["Cell Line/Type"] == "MDA-MB-468"]["Total proteins investigated"].values[0], 3)
+
+    def test_generate_dataset_statistics_workflow(self):
+        # Setup minimal image crate
+        img_dir = os.path.join(self.base_path, "blue")
+        os.makedirs(img_dir)
+        with open(os.path.join(img_dir, "img.tif"), "w") as f:
+            f.write("fake")
+
+        # Add metadata with a cell line
+        with open(os.path.join(self.base_path, "ro-crate-metadata.json"), "w") as f:
+            json.dump({"@graph": [{"name": "Study with Kolf2.1J"}]}, f)
+
+        # Add ENSEMBL file
+        df = pd.DataFrame({"ENSEMBL ID": ["ENSG000001"]})
+        df.to_csv(os.path.join(self.base_path, "table.tsv"), sep="\t", index=False)
+
+        # Patch gene symbol lookup to avoid mygene dependency
+        with patch("cellmaps_utils.statsgenerator.get_gene_symbols_from_ensembl_ids") as mock_mapper:
+            mock_mapper.return_value = {"ENSG000001": "TESTGENE"}
+            generate_dataset_statistics(images_crates=[self.base_path], interaction_crates=[])
+
+if __name__ == "__main__":
+    unittest.main()