ome-iris 0.0.3__py3-none-any.whl

OME_IRIS/__init__.py

@@ -0,0 +1,8 @@
+"""OME-IRIS package."""
+
+__all__ = ["__version__"]
+
+try:
+    from ._version import version as __version__
+except ImportError:  # pragma: no cover
+    __version__ = "0+unknown"

OME_IRIS/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '0.0.3'
+__version_tuple__ = version_tuple = (0, 0, 3)
+
+__commit_id__ = commit_id = None

OME_IRIS/clean.py

@@ -0,0 +1,9 @@
+from __future__ import annotations
+
+from pathlib import Path
+import shutil
+
+
+def clean_local_data(data_dir: Path) -> None:
+    if data_dir.exists():
+        shutil.rmtree(data_dir)

OME_IRIS/cli.py

@@ -0,0 +1,149 @@
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+
+from OME_IRIS.clean import clean_local_data
+from OME_IRIS.fetch import fetch_datasets
+from OME_IRIS.rocrate import export_rocrate_metadata
+from OME_IRIS.scaffold import scaffold_dataset_manifest
+from OME_IRIS.verify import verify_datasets
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="ome-iris", description="Fetch and verify OME-IRIS datasets"
+    )
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    fetch_cmd = sub.add_parser("fetch", help="Fetch dataset files")
+    fetch_cmd.add_argument("--dataset", dest="dataset_id")
+    fetch_cmd.add_argument("--tier", choices=["tiny", "small", "realistic"])
+    fetch_cmd.add_argument("--manifests-dir", default="src/OME_IRIS/data/datasets")
+    fetch_cmd.add_argument("--data-dir", default="data")
+    fetch_mode = fetch_cmd.add_mutually_exclusive_group()
+    fetch_mode.add_argument("--verbose", action="store_true")
+    fetch_mode.add_argument("--silent", action="store_true")
+
+    verify_cmd = sub.add_parser("verify", help="Verify local datasets")
+    verify_cmd.add_argument("--manifests-dir", default="src/OME_IRIS/data/datasets")
+    verify_cmd.add_argument("--data-dir", default="data")
+
+    clean_cmd = sub.add_parser("clean", help="Remove local fetched data")
+    clean_cmd.add_argument("--data-dir", default="data")
+
+    scaffold_cmd = sub.add_parser(
+        "scaffold",
+        help="Generate starter dataset manifest and CSV row from a source path",
+    )
+    scaffold_cmd.add_argument("--source-path", required=True)
+    scaffold_cmd.add_argument("--dataset-id")
+    scaffold_cmd.add_argument("--name", dest="dataset_name")
+    scaffold_cmd.add_argument(
+        "--tier", choices=["tiny", "small", "realistic"], default="small"
+    )
+    scaffold_cmd.add_argument("--license", dest="license_name", default="TBD")
+    scaffold_cmd.add_argument("--source-repository", default="")
+    scaffold_cmd.add_argument("--source-url", default="")
+    scaffold_cmd.add_argument("--include-directory-entry", action="store_true")
+    scaffold_cmd.add_argument("--directory-path", default="images")
+    scaffold_cmd.add_argument("--archive-format", choices=["zip", "tar"], default="zip")
+    scaffold_cmd.add_argument("--manifests-dir", default="src/OME_IRIS/data/datasets")
+    scaffold_cmd.add_argument("--catalog-csv", default="src/OME_IRIS/data/datasets.csv")
+    scaffold_cmd.add_argument("--append-csv", action="store_true")
+    scaffold_cmd.add_argument("--force", action="store_true")
+
+    rocrate_cmd = sub.add_parser(
+        "export-rocrate",
+        help="Export RO-Crate metadata for a dataset into the fetched dataset directory",
+    )
+    rocrate_cmd.add_argument("--dataset", dest="dataset_id", required=True)
+    rocrate_cmd.add_argument("--manifests-dir", default="src/OME_IRIS/data/datasets")
+    rocrate_cmd.add_argument("--data-dir", default="data")
+
+    return parser
+
+
+def main() -> int:
+    parser = build_parser()
+    args = parser.parse_args()
+
+    if args.command == "fetch":
+        result = fetch_datasets(
+            manifests_dir=Path(args.manifests_dir),
+            data_dir=Path(args.data_dir),
+            dataset_id=args.dataset_id,
+            tier=args.tier,
+            verbose=args.verbose,
+            silent=args.silent,
+        )
+        print(f"Downloaded: {result.downloaded}")
+        print(f"Skipped: {result.skipped}")
+        if result.downloaded_items:
+            print("Downloaded items:")
+            for item in result.downloaded_items:
+                print(f"- {item}")
+        if result.skipped_items:
+            print("Skipped items:")
+            for item in result.skipped_items:
+                print(f"- {item}")
+        if result.missing_urls:
+            print("Missing URLs:")
+            for item in result.missing_urls:
+                print(f"- {item}")
+        if result.failed:
+            print("Failed downloads:")
+            for item in result.failed:
+                print(f"- {item}")
+        return 0
+    if args.command == "clean":
+        clean_local_data(Path(args.data_dir))
+        print(f"Removed local data directory: {args.data_dir}")
+        return 0
+    if args.command == "scaffold":
+        result = scaffold_dataset_manifest(
+            source_path=args.source_path,
+            manifests_dir=Path(args.manifests_dir),
+            dataset_id=args.dataset_id,
+            dataset_name=args.dataset_name,
+            tier=args.tier,
+            license_name=args.license_name,
+            source_repository=args.source_repository,
+            source_url=args.source_url,
+            include_directory_entry=args.include_directory_entry,
+            directory_path=args.directory_path,
+            archive_format=args.archive_format,
+            append_csv=args.append_csv,
+            catalog_csv=Path(args.catalog_csv),
+            force=args.force,
+        )
+        print(f"Manifest created: {result.manifest_path}")
+        print("Suggested datasets.csv row:")
+        print(result.csv_row)
+        if args.append_csv:
+            print(f"Appended row to: {args.catalog_csv}")
+        return 0
+    if args.command == "export-rocrate":
+        out_path = export_rocrate_metadata(
+            manifests_dir=Path(args.manifests_dir),
+            dataset_id=args.dataset_id,
+            data_dir=Path(args.data_dir),
+        )
+        print(f"RO-Crate metadata written: {out_path}")
+        return 0
+
+    result = verify_datasets(
+        manifests_dir=Path(args.manifests_dir),
+        data_dir=Path(args.data_dir),
+    )
+    if result.ok:
+        print("Verification passed")
+        return 0
+    print("Verification failed")
+    for issue in result.issues:
+        print(f"- {issue}")
+    return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

OME_IRIS/data/dataset.schema.yaml

@@ -0,0 +1,104 @@
+id: https://ome-iris.org/schema/dataset
+name: ome_iris_dataset
+prefixes:
+  linkml: https://w3id.org/linkml/
+default_prefix: ome_iris_dataset
+default_range: string
+
+classes:
+  DatasetManifest:
+    attributes:
+      id:
+        required: true
+      name:
+        required: true
+      description:
+        required: true
+      tier:
+        required: true
+      license:
+        required: true
+      source_identifier:
+        required: true
+      source:
+        required: true
+        range: DatasetSource
+      formats:
+        required: true
+        multivalued: true
+      files:
+        required: true
+        multivalued: true
+        range: DatasetFile
+      relationships:
+        required: false
+        multivalued: true
+        range: DatasetRelationship
+      custom_metadata:
+        required: false
+        range: MetadataObject
+
+  DatasetSource:
+    attributes:
+      repository:
+        required: true
+      path:
+        required: true
+      url:
+        required: false
+      custom_metadata:
+        required: false
+        range: MetadataObject
+
+  DatasetFile:
+    attributes:
+      path:
+        required: true
+      kind:
+        required: false
+        permissible_values:
+          file:
+          directory:
+      url:
+        required: false
+      sha256:
+        required: false
+      archive_format:
+        required: false
+        permissible_values:
+          zip:
+          tar:
+      custom_metadata:
+        required: false
+        range: MetadataObject
+
+  DatasetRelationship:
+    attributes:
+      from:
+        required: true
+      to:
+        required: true
+      type:
+        required: true
+      rocrate_predicate:
+        required: true
+      via_columns:
+        required: false
+        multivalued: true
+      filename_patterns:
+        required: false
+        multivalued: true
+      derived_from_columns:
+        required: false
+        multivalued: true
+      custom_metadata:
+        required: false
+        range: MetadataObject
+
+  MetadataObject:
+    tree_root: true
+    attributes:
+      values:
+        multivalued: true
+        inlined: true
+        required: false

OME_IRIS/data/datasets/jump-plate-example.yaml

@@ -0,0 +1,45 @@
+---
+id: jump-plate-example
+name: JUMP plate BR00117006 (JUMP_plate_BR00117006) example
+description: >-
+  Plate-level cell painting benchmark subset focused on image IO and linking.
+tier: small
+license: CC-BY-4.0
+source_identifier: JUMP_plate_BR00117006
+source:
+  repository: https://github.com/cytomining/CytoDataFrame
+  path: tests/data/cytotable/JUMP_plate_BR00117006
+  url: >-
+    https://github.com/cytomining/CytoDataFrame/tree/main/tests/data/cytotable/JUMP_plate_BR00117006
+formats:
+  - parquet
+  - tiff
+files:
+  - path: BR00117006_shrunken.parquet
+    url: >-
+      https://github.com/cytomining/CytoDataFrame/raw/refs/heads/main/tests/data/cytotable/JUMP_plate_BR00117006/BR00117006_shrunken.parquet
+    custom_metadata:
+      role: profile_table
+  - path: images
+    kind: directory
+    url: >-
+      https://github.com/cytomining/CytoDataFrame/tree/main/tests/data/cytotable/JUMP_plate_BR00117006/images
+    custom_metadata:
+      role: image_bundle
+relationships:
+  - from: BR00117006_shrunken.parquet
+    to: images
+    type: links_to_images_by
+    via_columns:
+      - Image_PathName_OrigDNA
+      - Image_FileName_OrigDNA
+    rocrate_predicate: http://schema.org/associatedMedia
+  - from: BR00117006_shrunken.parquet
+    to: images
+    type: filename_key_mapping
+    via_columns:
+      - Metadata_Well
+      - Image_Metadata_Site
+    filename_patterns:
+      - "{well}_s{site}_w1*.tiff"
+    rocrate_predicate: http://schema.org/variableMeasured

OME_IRIS/data/datasets/nf1-cellpainting-shrunken.yaml

@@ -0,0 +1,68 @@
+---
+id: nf1-cellpainting-shrunken
+name: NF1 Cell Painting shrunken (NF1_cellpainting_data_shrunken) example
+description: >-
+  Small image-based profiling example for profile and image-link benchmarking.
+tier: small
+license: CC-BY-4.0
+source_identifier: NF1_cellpainting_data_shrunken
+source:
+  repository: https://github.com/cytomining/CytoDataFrame
+  path: tests/data/cytotable/NF1_cellpainting_data_shrunken
+  url: >-
+    https://github.com/cytomining/CytoDataFrame/tree/main/tests/data/cytotable/NF1_cellpainting_data_shrunken
+formats:
+  - parquet
+  - tiff
+files:
+  - path: profiles.parquet
+    url: >-
+      https://github.com/cytomining/CytoDataFrame/raw/refs/heads/main/tests/data/cytotable/NF1_cellpainting_data_shrunken/Plate_2_with_image_data_shrunken.parquet
+    custom_metadata:
+      role: profile_table
+  - path: images
+    kind: directory
+    url: >-
+      https://github.com/cytomining/CytoDataFrame/tree/main/tests/data/cytotable/NF1_cellpainting_data_shrunken/Plate_2_images
+    custom_metadata:
+      role: image_bundle
+  - path: masks
+    kind: directory
+    url: >-
+      https://github.com/cytomining/CytoDataFrame/tree/main/tests/data/cytotable/NF1_cellpainting_data_shrunken/Plate_2_masks
+    custom_metadata:
+      role: mask_bundle
+relationships:
+  - from: profiles.parquet
+    to: images
+    type: links_to_images_by
+    via_columns:
+      - Image_PathName_DAPI
+      - Image_FileName_DAPI
+      - Image_PathName_GFP
+      - Image_FileName_GFP
+      - Image_PathName_RFP
+      - Image_FileName_RFP
+    rocrate_predicate: http://schema.org/associatedMedia
+  - from: profiles.parquet
+    to: images
+    type: filename_key_mapping
+    via_columns:
+      - Metadata_Well
+      - Image_Metadata_Site
+    filename_patterns:
+      - "{well}_01_1_{site}_DAPI_001.tif"
+      - "{well}_01_2_{site}_GFP_001.tif"
+      - "{well}_01_3_{site}_RFP_001.tif"
+    rocrate_predicate: http://schema.org/variableMeasured
+  - from: images
+    to: masks
+    type: derived_masks_from
+    derived_from_columns:
+      - Image_FileName_DAPI
+      - Image_FileName_RFP
+    filename_patterns:
+      - "{well}_01_1_{site}_DAPI_001_MaskNuclei.tiff"
+      - "{well}_01_3_{site}_RFP_001_MaskCells.tiff"
+      - "{well}_01_3_{site}_RFP_001_MaskCytoplasm.tiff"
+    rocrate_predicate: http://www.w3.org/ns/prov#wasDerivedFrom

OME_IRIS/data/datasets/nuclei-3d.yaml

@@ -0,0 +1,61 @@
+---
+id: nuclei-3d
+name: 3D nuclei (CP_tutorial_3D_noise_nuclei_segmentation) example
+description: >-
+  Small 3D nuclei segmentation benchmark sample with image and mask files.
+tier: tiny
+license: CC-BY-4.0
+source_identifier: CP_tutorial_3D_noise_nuclei_segmentation
+source:
+  repository: https://github.com/cytomining/CytoDataFrame
+  path: tests/data/CP_tutorial_3D_noise_nuclei_segmentation
+  url: >-
+    https://github.com/cytomining/CytoDataFrame/tree/main/tests/data/CP_tutorial_3D_noise_nuclei_segmentation
+formats:
+  - csv
+  - tif
+  - tiff
+files:
+  - path: profiles.csv
+    url: >-
+      https://github.com/cytomining/CytoDataFrame/raw/refs/heads/main/tests/data/CP_tutorial_3D_noise_nuclei_segmentation/output/MyExpt_RealsizeNuclei.csv
+    custom_metadata:
+      role: profile_table
+  - path: images
+    kind: directory
+    url: >-
+      https://github.com/cytomining/CytoDataFrame/tree/main/tests/data/CP_tutorial_3D_noise_nuclei_segmentation/input
+    custom_metadata:
+      role: image_bundle
+  - path: masks
+    kind: directory
+    url: >-
+      https://github.com/cytomining/CytoDataFrame/tree/main/tests/data/CP_tutorial_3D_noise_nuclei_segmentation/output/masks
+    custom_metadata:
+      role: mask_bundle
+relationships:
+  - from: profiles.csv
+    to: images
+    type: links_to_images_by
+    via_columns:
+      - PathName_Nuclei
+      - FileName_Nuclei
+    rocrate_predicate: http://schema.org/associatedMedia
+  - from: profiles.csv
+    to: images
+    type: filename_key_mapping
+    via_columns:
+      - Metadata_Well
+      - Metadata_Site
+      - Metadata_Frame
+    filename_patterns:
+      - nuclei*_image.tif
+    rocrate_predicate: http://schema.org/variableMeasured
+  - from: images
+    to: masks
+    type: derived_masks_from
+    derived_from_columns:
+      - FileName_Nuclei
+    filename_patterns:
+      - "{image_stem}SegmentationMask.tiff"
+    rocrate_predicate: http://www.w3.org/ns/prov#wasDerivedFrom

OME_IRIS/data/datasets/pediatric-cancer-atlas.yaml

@@ -0,0 +1,65 @@
+---
+id: pediatric-cancer-atlas
+name: >-
+  Pediatric cancer atlas profiling (pediatric_cancer_atlas_profiling) example
+description: >-
+  Small atlas sample for metadata and profile workflow benchmarking.
+tier: realistic
+license: CC-BY-4.0
+source_identifier: pediatric_cancer_atlas_profiling
+source:
+  repository: https://github.com/cytomining/CytoDataFrame
+  path: tests/data/cytotable/pediatric_cancer_atlas_profiling
+  url: >-
+    https://github.com/cytomining/CytoDataFrame/tree/main/tests/data/cytotable/pediatric_cancer_atlas_profiling
+formats:
+  - parquet
+  - tiff
+files:
+  - path: profiles.parquet
+    url: >-
+      https://github.com/cytomining/CytoDataFrame/raw/refs/heads/main/tests/data/cytotable/pediatric_cancer_atlas_profiling/BR00143976_shrunken.parquet
+    custom_metadata:
+      role: profile_table
+  - path: images
+    kind: directory
+    url: >-
+      https://github.com/cytomining/CytoDataFrame/tree/main/tests/data/cytotable/pediatric_cancer_atlas_profiling/images/orig
+    custom_metadata:
+      role: image_bundle
+  - path: outlines
+    kind: directory
+    url: >-
+      https://github.com/cytomining/CytoDataFrame/tree/main/tests/data/cytotable/pediatric_cancer_atlas_profiling/images/outlines
+    custom_metadata:
+      role: outline_bundle
+relationships:
+  - from: profiles.parquet
+    to: images
+    type: links_to_images_by
+    via_columns:
+      - Image_Metadata_Well
+      - Image_Metadata_Site
+      - Image_FileName_OrigDNA
+      - Image_FileName_OrigAGP
+      - Image_FileName_OrigRNA
+      - Image_FileName_OrigER
+      - Image_FileName_OrigMito
+      - Image_FileName_OrigBrightfield
+    rocrate_predicate: http://schema.org/associatedMedia
+  - from: profiles.parquet
+    to: images
+    type: filename_key_mapping
+    via_columns:
+      - Image_Metadata_Well
+      - Image_Metadata_Site
+    filename_patterns:
+      - "r{well_row:02d}c{well_col:02d}f{site:02d}p01-ch{channel}sk1fk1fl1.tiff"
+    rocrate_predicate: http://schema.org/variableMeasured
+  - from: images
+    to: outlines
+    type: derived_outlines_from
+    filename_patterns:
+      - CellsOutlines_BR00143976_{well}_{site}.tiff
+      - NucleiOutlines_BR00143976_{well}_{site}.tiff
+    rocrate_predicate: http://www.w3.org/ns/prov#wasDerivedFrom

OME_IRIS/data/datasets.csv

@@ -0,0 +1,5 @@
+id,name,tier,formats,benchmark_roles,license,source
+nuclei-3d,3D nuclei (CP_tutorial_3D_noise_nuclei_segmentation) example,tiny,"csv,tif,tiff",image_read|mask_read,CC-BY-4.0,CytoDataFrame
+nf1-cellpainting-shrunken,NF1 Cell Painting shrunken (NF1_cellpainting_data_shrunken) example,small,"parquet,tiff",profile_read|image_profile_link,CC-BY-4.0,CytoDataFrame
+jump-plate-example,JUMP plate BR00117006 (JUMP_plate_BR00117006) example,small,"parquet,tiff",image_read|image_profile_link,CC-BY-4.0,CytoDataFrame
+pediatric-cancer-atlas,Pediatric cancer atlas profiling (pediatric_cancer_atlas_profiling) example,realistic,"parquet,tiff",profile_read|metadata_read,CC-BY-4.0,CytoDataFrame