fair-metadata-integration 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fair_metadata_integration-0.1.0/LICENSE +21 -0
- fair_metadata_integration-0.1.0/PKG-INFO +124 -0
- fair_metadata_integration-0.1.0/README.md +102 -0
- fair_metadata_integration-0.1.0/pyproject.toml +42 -0
- fair_metadata_integration-0.1.0/setup.cfg +4 -0
- fair_metadata_integration-0.1.0/src/fair_metadata_integration/__init__.py +3 -0
- fair_metadata_integration-0.1.0/src/fair_metadata_integration/metadata_integration.py +95 -0
- fair_metadata_integration-0.1.0/src/fair_metadata_integration.egg-info/PKG-INFO +124 -0
- fair_metadata_integration-0.1.0/src/fair_metadata_integration.egg-info/SOURCES.txt +11 -0
- fair_metadata_integration-0.1.0/src/fair_metadata_integration.egg-info/dependency_links.txt +1 -0
- fair_metadata_integration-0.1.0/src/fair_metadata_integration.egg-info/requires.txt +4 -0
- fair_metadata_integration-0.1.0/src/fair_metadata_integration.egg-info/top_level.txt +1 -0
- fair_metadata_integration-0.1.0/tests/test_metadata_integration.py +246 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 NL-BioImaging
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: fair-metadata-integration
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Integrate and package all metadata into RO-Crate, linking to other metadata files
|
|
5
|
+
Author-email: Joost de Folter <folterj@gmail.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/NL-BioImaging/fair-metadata-integration
|
|
8
|
+
Project-URL: Repository, https://github.com/NL-BioImaging/fair-metadata-integration
|
|
9
|
+
Keywords: RO-Crate,FAIR,metadata,bioimaging,OME-Zarr
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Science/Research
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
|
|
15
|
+
Requires-Python: >=3.12
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
License-File: LICENSE
|
|
18
|
+
Requires-Dist: rocrate>=0.15.1
|
|
19
|
+
Provides-Extra: test
|
|
20
|
+
Requires-Dist: pytest>=7.0; extra == "test"
|
|
21
|
+
Dynamic: license-file
|
|
22
|
+
|
|
23
|
+
# fair-metadata-integration
|
|
24
|
+
|
|
25
|
+
Integrate and package all metadata into RO-Crate, linking to other metadata files.
|
|
26
|
+
|
|
27
|
+
Given a [Bilayers](https://github.com/bilayer-containers/bilayers) workflow
|
|
28
|
+
specification and the image data an analysis consumed and produced, this package
|
|
29
|
+
writes a [Workflow RO-Crate](https://w3id.org/workflowhub/workflow-ro-crate/1.0):
|
|
30
|
+
an `ro-crate-metadata.json` describing the run, alongside copies of the files it
|
|
31
|
+
describes.
|
|
32
|
+
|
|
33
|
+
## Installation
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
pip install fair-metadata-integration
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
Requires Python 3.12 or newer. [`rocrate`](https://pypi.org/project/rocrate/) is
|
|
40
|
+
installed automatically.
|
|
41
|
+
|
|
42
|
+
## Usage
|
|
43
|
+
|
|
44
|
+
```python
|
|
45
|
+
from fair_metadata_integration import create_ro_crate
|
|
46
|
+
|
|
47
|
+
create_ro_crate(
|
|
48
|
+
dest_path="crate",
|
|
49
|
+
workflow_schema_filename="config.yaml",
|
|
50
|
+
input_path="input.ome.zarr",
|
|
51
|
+
output_paths=["output.ome.zarr"],
|
|
52
|
+
)
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
- `dest_path` — directory the crate is written to, created if it does not exist.
|
|
56
|
+
- `workflow_schema_filename` — the Bilayers `config.yaml` describing the workflow.
|
|
57
|
+
- `input_path` — the input image (an OME-Zarr directory or a TIFF file).
|
|
58
|
+
- `output_paths` — the images the workflow produced. Each gets its own parameter.
|
|
59
|
+
|
|
60
|
+
The source files are copied into the crate, so `dest_path` ends up
|
|
61
|
+
self-contained:
|
|
62
|
+
|
|
63
|
+
```
|
|
64
|
+
crate/
|
|
65
|
+
├── config.yaml
|
|
66
|
+
├── input.ome.zarr/
|
|
67
|
+
├── output.ome.zarr/
|
|
68
|
+
└── ro-crate-metadata.json
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
### What the crate contains
|
|
72
|
+
|
|
73
|
+
The workflow specification is the crate's `mainEntity`, typed as a
|
|
74
|
+
`ComputationalWorkflow`, and the metadata descriptor declares the Workflow
|
|
75
|
+
RO-Crate profile:
|
|
76
|
+
|
|
77
|
+
```json
|
|
78
|
+
{
|
|
79
|
+
"@id": "ro-crate-metadata.json",
|
|
80
|
+
"@type": "CreativeWork",
|
|
81
|
+
"about": { "@id": "./" },
|
|
82
|
+
"conformsTo": [
|
|
83
|
+
{ "@id": "https://w3id.org/ro/crate/1.2" },
|
|
84
|
+
{ "@id": "https://w3id.org/workflowhub/workflow-ro-crate/1.0" }
|
|
85
|
+
]
|
|
86
|
+
}
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
The workflow's `programmingLanguage` is a `ComputerLanguage` entity for the
|
|
90
|
+
Bilayers specification, identified by its Zenodo concept DOI
|
|
91
|
+
[`10.5281/zenodo.17652333`](https://doi.org/10.5281/zenodo.17652333), which
|
|
92
|
+
resolves to the latest release of the spec.
|
|
93
|
+
|
|
94
|
+
Inputs and outputs are modelled as `FormalParameter` entities carrying an
|
|
95
|
+
[EDAM](https://edamontology.org/) format IRI — `format_3915` for Zarr,
|
|
96
|
+
`format_3591` for TIFF. Each image dataset points back at the parameter it
|
|
97
|
+
realises via `exampleOfWork`:
|
|
98
|
+
|
|
99
|
+
```json
|
|
100
|
+
{
|
|
101
|
+
"@id": "input.ome.zarr/",
|
|
102
|
+
"@type": "Dataset",
|
|
103
|
+
"additionalType": "http://edamontology.org/format_3915",
|
|
104
|
+
"exampleOfWork": { "@id": "#input" }
|
|
105
|
+
}
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
The crate's JSON-LD context is extended with terms for describing biological
|
|
109
|
+
imaging metadata — `biosample`, `specimen`, `organism_classification`,
|
|
110
|
+
`preparation_method`, `channel` and others — so those properties can be added to
|
|
111
|
+
the crate's entities.
|
|
112
|
+
|
|
113
|
+
## Development
|
|
114
|
+
|
|
115
|
+
```bash
|
|
116
|
+
git clone https://github.com/NL-BioImaging/fair-metadata-integration.git
|
|
117
|
+
cd fair-metadata-integration
|
|
118
|
+
pip install -e ".[test]"
|
|
119
|
+
pytest
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
## License
|
|
123
|
+
|
|
124
|
+
MIT
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
# fair-metadata-integration
|
|
2
|
+
|
|
3
|
+
Integrate and package all metadata into RO-Crate, linking to other metadata files.
|
|
4
|
+
|
|
5
|
+
Given a [Bilayers](https://github.com/bilayer-containers/bilayers) workflow
|
|
6
|
+
specification and the image data an analysis consumed and produced, this package
|
|
7
|
+
writes a [Workflow RO-Crate](https://w3id.org/workflowhub/workflow-ro-crate/1.0):
|
|
8
|
+
an `ro-crate-metadata.json` describing the run, alongside copies of the files it
|
|
9
|
+
describes.
|
|
10
|
+
|
|
11
|
+
## Installation
|
|
12
|
+
|
|
13
|
+
```bash
|
|
14
|
+
pip install fair-metadata-integration
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
Requires Python 3.12 or newer. [`rocrate`](https://pypi.org/project/rocrate/) is
|
|
18
|
+
installed automatically.
|
|
19
|
+
|
|
20
|
+
## Usage
|
|
21
|
+
|
|
22
|
+
```python
|
|
23
|
+
from fair_metadata_integration import create_ro_crate
|
|
24
|
+
|
|
25
|
+
create_ro_crate(
|
|
26
|
+
dest_path="crate",
|
|
27
|
+
workflow_schema_filename="config.yaml",
|
|
28
|
+
input_path="input.ome.zarr",
|
|
29
|
+
output_paths=["output.ome.zarr"],
|
|
30
|
+
)
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
- `dest_path` — directory the crate is written to, created if it does not exist.
|
|
34
|
+
- `workflow_schema_filename` — the Bilayers `config.yaml` describing the workflow.
|
|
35
|
+
- `input_path` — the input image (an OME-Zarr directory or a TIFF file).
|
|
36
|
+
- `output_paths` — the images the workflow produced. Each gets its own parameter.
|
|
37
|
+
|
|
38
|
+
The source files are copied into the crate, so `dest_path` ends up
|
|
39
|
+
self-contained:
|
|
40
|
+
|
|
41
|
+
```
|
|
42
|
+
crate/
|
|
43
|
+
├── config.yaml
|
|
44
|
+
├── input.ome.zarr/
|
|
45
|
+
├── output.ome.zarr/
|
|
46
|
+
└── ro-crate-metadata.json
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
### What the crate contains
|
|
50
|
+
|
|
51
|
+
The workflow specification is the crate's `mainEntity`, typed as a
|
|
52
|
+
`ComputationalWorkflow`, and the metadata descriptor declares the Workflow
|
|
53
|
+
RO-Crate profile:
|
|
54
|
+
|
|
55
|
+
```json
|
|
56
|
+
{
|
|
57
|
+
"@id": "ro-crate-metadata.json",
|
|
58
|
+
"@type": "CreativeWork",
|
|
59
|
+
"about": { "@id": "./" },
|
|
60
|
+
"conformsTo": [
|
|
61
|
+
{ "@id": "https://w3id.org/ro/crate/1.2" },
|
|
62
|
+
{ "@id": "https://w3id.org/workflowhub/workflow-ro-crate/1.0" }
|
|
63
|
+
]
|
|
64
|
+
}
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
The workflow's `programmingLanguage` is a `ComputerLanguage` entity for the
|
|
68
|
+
Bilayers specification, identified by its Zenodo concept DOI
|
|
69
|
+
[`10.5281/zenodo.17652333`](https://doi.org/10.5281/zenodo.17652333), which
|
|
70
|
+
resolves to the latest release of the spec.
|
|
71
|
+
|
|
72
|
+
Inputs and outputs are modelled as `FormalParameter` entities carrying an
|
|
73
|
+
[EDAM](https://edamontology.org/) format IRI — `format_3915` for Zarr,
|
|
74
|
+
`format_3591` for TIFF. Each image dataset points back at the parameter it
|
|
75
|
+
realises via `exampleOfWork`:
|
|
76
|
+
|
|
77
|
+
```json
|
|
78
|
+
{
|
|
79
|
+
"@id": "input.ome.zarr/",
|
|
80
|
+
"@type": "Dataset",
|
|
81
|
+
"additionalType": "http://edamontology.org/format_3915",
|
|
82
|
+
"exampleOfWork": { "@id": "#input" }
|
|
83
|
+
}
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
The crate's JSON-LD context is extended with terms for describing biological
|
|
87
|
+
imaging metadata — `biosample`, `specimen`, `organism_classification`,
|
|
88
|
+
`preparation_method`, `channel` and others — so those properties can be added to
|
|
89
|
+
the crate's entities.
|
|
90
|
+
|
|
91
|
+
## Development
|
|
92
|
+
|
|
93
|
+
```bash
|
|
94
|
+
git clone https://github.com/NL-BioImaging/fair-metadata-integration.git
|
|
95
|
+
cd fair-metadata-integration
|
|
96
|
+
pip install -e ".[test]"
|
|
97
|
+
pytest
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
## License
|
|
101
|
+
|
|
102
|
+
MIT
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "fair-metadata-integration"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Integrate and package all metadata into RO-Crate, linking to other metadata files"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
license-files = ["LICENSE"]
|
|
12
|
+
requires-python = ">=3.12"
|
|
13
|
+
authors = [
|
|
14
|
+
{ name = "Joost de Folter", email = "folterj@gmail.com" },
|
|
15
|
+
]
|
|
16
|
+
keywords = ["RO-Crate", "FAIR", "metadata", "bioimaging", "OME-Zarr"]
|
|
17
|
+
classifiers = [
|
|
18
|
+
"Development Status :: 3 - Alpha",
|
|
19
|
+
"Intended Audience :: Science/Research",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Programming Language :: Python :: 3.12",
|
|
22
|
+
"Topic :: Scientific/Engineering :: Bio-Informatics",
|
|
23
|
+
]
|
|
24
|
+
dependencies = [
|
|
25
|
+
"rocrate>=0.15.1",
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
[project.optional-dependencies]
|
|
29
|
+
test = ["pytest>=7.0"]
|
|
30
|
+
|
|
31
|
+
[project.urls]
|
|
32
|
+
Homepage = "https://github.com/NL-BioImaging/fair-metadata-integration"
|
|
33
|
+
Repository = "https://github.com/NL-BioImaging/fair-metadata-integration"
|
|
34
|
+
|
|
35
|
+
[tool.setuptools.packages.find]
|
|
36
|
+
where = ["src"]
|
|
37
|
+
|
|
38
|
+
[tool.pytest.ini_options]
|
|
39
|
+
pythonpath = ["src"]
|
|
40
|
+
testpaths = ["tests"]
|
|
41
|
+
# -s disables output capture so print() from passing tests is visible
|
|
42
|
+
addopts = "-s"
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import os.path
|
|
2
|
+
from rocrate.rocrate import ROCrate
|
|
3
|
+
from rocrate.model import ComputerLanguage, ContextEntity
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def create_ro_crate(dest_path, workflow_schema_filename, input_path, output_paths=[]):
|
|
7
|
+
crate = ROCrate()
|
|
8
|
+
|
|
9
|
+
crate.metadata.extra_terms = {
|
|
10
|
+
"organism_classification": "https://schema.org/taxonomicRange",
|
|
11
|
+
"BioChemEntity": "https://schema.org/BioChemEntity",
|
|
12
|
+
"channel": "https://www.openmicroscopy.org/Schemas/Documentation/Generated/OME-2016-06/ome_xsd.html#Channel",
|
|
13
|
+
"obo": "http://purl.obolibrary.org/obo/",
|
|
14
|
+
"FBcv": "http://ontobee.org/ontology/FBcv/",
|
|
15
|
+
"acquisiton_method": {
|
|
16
|
+
"@reverse": "https://schema.org/result",
|
|
17
|
+
"@type": "@id",
|
|
18
|
+
},
|
|
19
|
+
"biological_entity": "https://schema.org/about",
|
|
20
|
+
"biosample": "http://purl.obolibrary.org/obo/OBI_0002648",
|
|
21
|
+
"preparation_method": "https://www.wikidata.org/wiki/Property:P1537",
|
|
22
|
+
"specimen": "http://purl.obolibrary.org/obo/HSO_0000308",
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
output_entities = []
|
|
26
|
+
for image_path in output_paths:
|
|
27
|
+
rel_path = os.path.basename(image_path.rstrip('\\/'))
|
|
28
|
+
output_entities.append(crate.add_dataset(
|
|
29
|
+
source=image_path, dest_path=rel_path,
|
|
30
|
+
properties=get_additional_filetype_property(image_path)))
|
|
31
|
+
|
|
32
|
+
input_entity = crate.add_dataset(source=input_path,
|
|
33
|
+
dest_path=os.path.basename(input_path.rstrip('\\/')),
|
|
34
|
+
properties=get_additional_filetype_property(input_path))
|
|
35
|
+
|
|
36
|
+
# main=True sets the root dataset's mainEntity and declares the Workflow RO-Crate
|
|
37
|
+
# profile in conformsTo (on the metadata descriptor for rocrate < 0.16, on the
|
|
38
|
+
# root dataset for rocrate >= 0.16).
|
|
39
|
+
workflow = crate.add_workflow(source=workflow_schema_filename,
|
|
40
|
+
dest_path=os.path.basename(workflow_schema_filename),
|
|
41
|
+
main=True, lang=get_bilayers_language(crate))
|
|
42
|
+
|
|
43
|
+
# The Workflow RO-Crate profile ranges input/output over FormalParameter rather
|
|
44
|
+
# than over the data entities themselves; each data entity is linked back to its
|
|
45
|
+
# parameter with exampleOfWork.
|
|
46
|
+
workflow['input'] = [add_formal_parameter(crate, input_entity, 'input')]
|
|
47
|
+
if output_entities:
|
|
48
|
+
workflow['output'] = [
|
|
49
|
+
add_formal_parameter(crate, entity, 'output' if i == 0 else f'output_{i + 1}')
|
|
50
|
+
for i, entity in enumerate(output_entities)
|
|
51
|
+
]
|
|
52
|
+
|
|
53
|
+
crate.write(dest_path)
|
|
54
|
+
return crate
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def get_additional_filetype_property(path):
|
|
58
|
+
ext = os.path.splitext(path)[1]
|
|
59
|
+
ref = None
|
|
60
|
+
if '.zar' in ext:
|
|
61
|
+
ref = 'http://edamontology.org/format_3915'
|
|
62
|
+
elif '.tif' in ext:
|
|
63
|
+
ref = 'http://edamontology.org/format_3591'
|
|
64
|
+
if ref:
|
|
65
|
+
return {'additionalType': ref}
|
|
66
|
+
return None
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def get_bilayers_language(crate):
|
|
70
|
+
"""Return the ComputerLanguage entity for a bilayers workflow specification.
|
|
71
|
+
|
|
72
|
+
Identified by the Zenodo concept DOI, which is version-independent and
|
|
73
|
+
resolves to the latest release of the specification.
|
|
74
|
+
"""
|
|
75
|
+
return crate.add(ComputerLanguage(
|
|
76
|
+
crate,
|
|
77
|
+
identifier='https://doi.org/10.5281/zenodo.17652333',
|
|
78
|
+
properties={
|
|
79
|
+
'name': 'Bilayers',
|
|
80
|
+
'alternateName': 'Bilayers workflow specification',
|
|
81
|
+
'identifier': {'@id': 'https://doi.org/10.5281/zenodo.17652333'},
|
|
82
|
+
'url': {'@id': 'https://github.com/bilayer-containers/bilayers'},
|
|
83
|
+
},
|
|
84
|
+
))
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def add_formal_parameter(crate, data_entity, name):
|
|
88
|
+
"""Add a FormalParameter describing *data_entity* and link the two together."""
|
|
89
|
+
properties = {'@type': 'FormalParameter', 'name': name}
|
|
90
|
+
format_ref = data_entity.get('additionalType')
|
|
91
|
+
if format_ref:
|
|
92
|
+
properties['format'] = {'@id': format_ref}
|
|
93
|
+
parameter = crate.add(ContextEntity(crate, f'#{name}', properties))
|
|
94
|
+
data_entity['exampleOfWork'] = parameter
|
|
95
|
+
return parameter
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: fair-metadata-integration
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Integrate and package all metadata into RO-Crate, linking to other metadata files
|
|
5
|
+
Author-email: Joost de Folter <folterj@gmail.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/NL-BioImaging/fair-metadata-integration
|
|
8
|
+
Project-URL: Repository, https://github.com/NL-BioImaging/fair-metadata-integration
|
|
9
|
+
Keywords: RO-Crate,FAIR,metadata,bioimaging,OME-Zarr
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Science/Research
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
|
|
15
|
+
Requires-Python: >=3.12
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
License-File: LICENSE
|
|
18
|
+
Requires-Dist: rocrate>=0.15.1
|
|
19
|
+
Provides-Extra: test
|
|
20
|
+
Requires-Dist: pytest>=7.0; extra == "test"
|
|
21
|
+
Dynamic: license-file
|
|
22
|
+
|
|
23
|
+
# fair-metadata-integration
|
|
24
|
+
|
|
25
|
+
Integrate and package all metadata into RO-Crate, linking to other metadata files.
|
|
26
|
+
|
|
27
|
+
Given a [Bilayers](https://github.com/bilayer-containers/bilayers) workflow
|
|
28
|
+
specification and the image data an analysis consumed and produced, this package
|
|
29
|
+
writes a [Workflow RO-Crate](https://w3id.org/workflowhub/workflow-ro-crate/1.0):
|
|
30
|
+
an `ro-crate-metadata.json` describing the run, alongside copies of the files it
|
|
31
|
+
describes.
|
|
32
|
+
|
|
33
|
+
## Installation
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
pip install fair-metadata-integration
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
Requires Python 3.12 or newer. [`rocrate`](https://pypi.org/project/rocrate/) is
|
|
40
|
+
installed automatically.
|
|
41
|
+
|
|
42
|
+
## Usage
|
|
43
|
+
|
|
44
|
+
```python
|
|
45
|
+
from fair_metadata_integration import create_ro_crate
|
|
46
|
+
|
|
47
|
+
create_ro_crate(
|
|
48
|
+
dest_path="crate",
|
|
49
|
+
workflow_schema_filename="config.yaml",
|
|
50
|
+
input_path="input.ome.zarr",
|
|
51
|
+
output_paths=["output.ome.zarr"],
|
|
52
|
+
)
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
- `dest_path` — directory the crate is written to, created if it does not exist.
|
|
56
|
+
- `workflow_schema_filename` — the Bilayers `config.yaml` describing the workflow.
|
|
57
|
+
- `input_path` — the input image (an OME-Zarr directory or a TIFF file).
|
|
58
|
+
- `output_paths` — the images the workflow produced. Each gets its own parameter.
|
|
59
|
+
|
|
60
|
+
The source files are copied into the crate, so `dest_path` ends up
|
|
61
|
+
self-contained:
|
|
62
|
+
|
|
63
|
+
```
|
|
64
|
+
crate/
|
|
65
|
+
├── config.yaml
|
|
66
|
+
├── input.ome.zarr/
|
|
67
|
+
├── output.ome.zarr/
|
|
68
|
+
└── ro-crate-metadata.json
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
### What the crate contains
|
|
72
|
+
|
|
73
|
+
The workflow specification is the crate's `mainEntity`, typed as a
|
|
74
|
+
`ComputationalWorkflow`, and the metadata descriptor declares the Workflow
|
|
75
|
+
RO-Crate profile:
|
|
76
|
+
|
|
77
|
+
```json
|
|
78
|
+
{
|
|
79
|
+
"@id": "ro-crate-metadata.json",
|
|
80
|
+
"@type": "CreativeWork",
|
|
81
|
+
"about": { "@id": "./" },
|
|
82
|
+
"conformsTo": [
|
|
83
|
+
{ "@id": "https://w3id.org/ro/crate/1.2" },
|
|
84
|
+
{ "@id": "https://w3id.org/workflowhub/workflow-ro-crate/1.0" }
|
|
85
|
+
]
|
|
86
|
+
}
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
The workflow's `programmingLanguage` is a `ComputerLanguage` entity for the
|
|
90
|
+
Bilayers specification, identified by its Zenodo concept DOI
|
|
91
|
+
[`10.5281/zenodo.17652333`](https://doi.org/10.5281/zenodo.17652333), which
|
|
92
|
+
resolves to the latest release of the spec.
|
|
93
|
+
|
|
94
|
+
Inputs and outputs are modelled as `FormalParameter` entities carrying an
|
|
95
|
+
[EDAM](https://edamontology.org/) format IRI — `format_3915` for Zarr,
|
|
96
|
+
`format_3591` for TIFF. Each image dataset points back at the parameter it
|
|
97
|
+
realises via `exampleOfWork`:
|
|
98
|
+
|
|
99
|
+
```json
|
|
100
|
+
{
|
|
101
|
+
"@id": "input.ome.zarr/",
|
|
102
|
+
"@type": "Dataset",
|
|
103
|
+
"additionalType": "http://edamontology.org/format_3915",
|
|
104
|
+
"exampleOfWork": { "@id": "#input" }
|
|
105
|
+
}
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
The crate's JSON-LD context is extended with terms for describing biological
|
|
109
|
+
imaging metadata — `biosample`, `specimen`, `organism_classification`,
|
|
110
|
+
`preparation_method`, `channel` and others — so those properties can be added to
|
|
111
|
+
the crate's entities.
|
|
112
|
+
|
|
113
|
+
## Development
|
|
114
|
+
|
|
115
|
+
```bash
|
|
116
|
+
git clone https://github.com/NL-BioImaging/fair-metadata-integration.git
|
|
117
|
+
cd fair-metadata-integration
|
|
118
|
+
pip install -e ".[test]"
|
|
119
|
+
pytest
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
## License
|
|
123
|
+
|
|
124
|
+
MIT
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
src/fair_metadata_integration/__init__.py
|
|
5
|
+
src/fair_metadata_integration/metadata_integration.py
|
|
6
|
+
src/fair_metadata_integration.egg-info/PKG-INFO
|
|
7
|
+
src/fair_metadata_integration.egg-info/SOURCES.txt
|
|
8
|
+
src/fair_metadata_integration.egg-info/dependency_links.txt
|
|
9
|
+
src/fair_metadata_integration.egg-info/requires.txt
|
|
10
|
+
src/fair_metadata_integration.egg-info/top_level.txt
|
|
11
|
+
tests/test_metadata_integration.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
fair_metadata_integration
|
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import os
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
import pytest
|
|
5
|
+
|
|
6
|
+
from fair_metadata_integration.metadata_integration import create_ro_crate
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
BILAYERS_CONFIG = """\
|
|
10
|
+
citations:
|
|
11
|
+
- name: "Empanada"
|
|
12
|
+
doi: "10.1083/jcb.202208005"
|
|
13
|
+
license: "Apache-2.0"
|
|
14
|
+
description: "Panoptic segmentation of organelles in volume electron microscopy"
|
|
15
|
+
|
|
16
|
+
docker_image:
|
|
17
|
+
org: "czii"
|
|
18
|
+
name: "empanada-napari"
|
|
19
|
+
tag: "latest"
|
|
20
|
+
platform: "linux/amd64"
|
|
21
|
+
|
|
22
|
+
algorithm_folder_name: "empanada_inference"
|
|
23
|
+
|
|
24
|
+
exec_function:
|
|
25
|
+
name: "generate_cli_command"
|
|
26
|
+
cli_command: "python -m empanada.inference"
|
|
27
|
+
hidden_args: []
|
|
28
|
+
|
|
29
|
+
inputs:
|
|
30
|
+
- name: input_image
|
|
31
|
+
type: image
|
|
32
|
+
cli_tag: "--image"
|
|
33
|
+
cli_order: 1
|
|
34
|
+
|
|
35
|
+
outputs:
|
|
36
|
+
- name: segmentation
|
|
37
|
+
type: image
|
|
38
|
+
cli_tag: "--output"
|
|
39
|
+
cli_order: 2
|
|
40
|
+
|
|
41
|
+
parameters:
|
|
42
|
+
- name: model
|
|
43
|
+
type: string
|
|
44
|
+
cli_tag: "--model"
|
|
45
|
+
default: "MitoNet_v1_mini"
|
|
46
|
+
|
|
47
|
+
display_only: []
|
|
48
|
+
"""
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def make_bilayers_config(path: Path) -> Path:
|
|
52
|
+
"""Write a minimal bilayers config.yaml to *path* and return it."""
|
|
53
|
+
config = path / "config.yaml"
|
|
54
|
+
config.write_text(BILAYERS_CONFIG)
|
|
55
|
+
return config
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def make_ome_zarr_dirs(path: Path) -> tuple[Path, Path]:
|
|
59
|
+
"""Create input/output ome-zarr directories under *path* and return them."""
|
|
60
|
+
input_zarr = path / "input.ome.zarr"
|
|
61
|
+
output_zarr = path / "output.ome.zarr"
|
|
62
|
+
# Nested content, so tests can check the whole tree is copied into the crate.
|
|
63
|
+
(input_zarr / "0").mkdir(parents=True)
|
|
64
|
+
(input_zarr / ".zattrs").write_text('{"multiscales": []}')
|
|
65
|
+
(input_zarr / "0" / "0.0.0").write_bytes(b"\x01\x02\x03")
|
|
66
|
+
output_zarr.mkdir()
|
|
67
|
+
(output_zarr / ".zattrs").write_text('{"segmentation": true}')
|
|
68
|
+
return input_zarr, output_zarr
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class TestCreateRoCrate:
|
|
72
|
+
@pytest.fixture()
|
|
73
|
+
def bilayers_config(self, tmp_path: Path) -> Path:
|
|
74
|
+
return make_bilayers_config(tmp_path)
|
|
75
|
+
|
|
76
|
+
@pytest.fixture()
|
|
77
|
+
def ome_zarr_dirs(self, tmp_path: Path) -> tuple[Path, Path]:
|
|
78
|
+
return make_ome_zarr_dirs(tmp_path)
|
|
79
|
+
|
|
80
|
+
def test_create_ro_crate(self, tmp_path, bilayers_config, ome_zarr_dirs):
|
|
81
|
+
"""create_ro_crate writes a valid RO-Crate with all expected entities."""
|
|
82
|
+
input_zarr, output_zarr = ome_zarr_dirs
|
|
83
|
+
dest = tmp_path / "crate"
|
|
84
|
+
dest.mkdir()
|
|
85
|
+
|
|
86
|
+
create_ro_crate(
|
|
87
|
+
dest_path=str(dest),
|
|
88
|
+
workflow_schema_filename=str(bilayers_config),
|
|
89
|
+
input_path=str(input_zarr),
|
|
90
|
+
output_paths=[str(output_zarr)],
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
assert (dest / "ro-crate-metadata.json").exists()
|
|
94
|
+
|
|
95
|
+
metadata = json.loads((dest / "ro-crate-metadata.json").read_text())
|
|
96
|
+
print("\n--- ro-crate-metadata.json ---")
|
|
97
|
+
print(json.dumps(metadata, indent=2))
|
|
98
|
+
|
|
99
|
+
ids = self._read_ids(dest)
|
|
100
|
+
# rocrate normalises paths to forward-slashes with a trailing '/'.
|
|
101
|
+
assert any(eid == "output.ome.zarr/" for eid in ids), (
|
|
102
|
+
f"Output zarr not found in crate entities: {ids}"
|
|
103
|
+
)
|
|
104
|
+
# Entities must live inside the crate root, never above it.
|
|
105
|
+
assert not any(eid.startswith("../") for eid in ids), (
|
|
106
|
+
f"Crate contains entities outside the crate root: {ids}"
|
|
107
|
+
)
|
|
108
|
+
assert any("config.yaml" in eid for eid in ids), (
|
|
109
|
+
f"Bilayers config.yaml not found in crate entities: {ids}"
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
def test_workflow_ro_crate_profile(self, tmp_path, bilayers_config, ome_zarr_dirs):
|
|
113
|
+
"""The crate declares the Workflow RO-Crate profile correctly."""
|
|
114
|
+
input_zarr, output_zarr = ome_zarr_dirs
|
|
115
|
+
dest = tmp_path / "crate"
|
|
116
|
+
dest.mkdir()
|
|
117
|
+
|
|
118
|
+
create_ro_crate(
|
|
119
|
+
dest_path=str(dest),
|
|
120
|
+
workflow_schema_filename=str(bilayers_config),
|
|
121
|
+
input_path=str(input_zarr),
|
|
122
|
+
output_paths=[str(output_zarr)],
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
entities = self._read_entities(dest)
|
|
126
|
+
|
|
127
|
+
# The crate must declare the Workflow RO-Crate profile. rocrate < 0.16
|
|
128
|
+
# (RO-Crate 1.1) lists it on the metadata descriptor; rocrate >= 0.16
|
|
129
|
+
# (RO-Crate 1.2+) lists it on the root dataset.
|
|
130
|
+
conforms_to = set()
|
|
131
|
+
for entity_id in ("ro-crate-metadata.json", "./"):
|
|
132
|
+
refs = entities[entity_id].get("conformsTo", [])
|
|
133
|
+
if isinstance(refs, dict):
|
|
134
|
+
refs = [refs]
|
|
135
|
+
conforms_to.update(ref["@id"] for ref in refs)
|
|
136
|
+
assert any(
|
|
137
|
+
uri.startswith("https://w3id.org/workflowhub/workflow-ro-crate/")
|
|
138
|
+
for uri in conforms_to
|
|
139
|
+
), conforms_to
|
|
140
|
+
|
|
141
|
+
# The root dataset must point at the workflow as its main entity.
|
|
142
|
+
assert entities["./"]["mainEntity"] == {"@id": "config.yaml"}
|
|
143
|
+
|
|
144
|
+
workflow = entities["config.yaml"]
|
|
145
|
+
assert "ComputationalWorkflow" in workflow["@type"]
|
|
146
|
+
|
|
147
|
+
# input/output must reference FormalParameter entities, not the datasets.
|
|
148
|
+
for prop in ("input", "output"):
|
|
149
|
+
for ref in workflow[prop]:
|
|
150
|
+
assert entities[ref["@id"]]["@type"] == "FormalParameter", (
|
|
151
|
+
f"workflow {prop} '{ref['@id']}' is not a FormalParameter"
|
|
152
|
+
)
|
|
153
|
+
|
|
154
|
+
# Each dataset links back to the parameter it is an example of.
|
|
155
|
+
assert entities["input.ome.zarr/"]["exampleOfWork"] == {"@id": "#input"}
|
|
156
|
+
assert entities["output.ome.zarr/"]["exampleOfWork"] == {"@id": "#output"}
|
|
157
|
+
|
|
158
|
+
# The language must describe bilayers, not CWL, and must be identified by
|
|
159
|
+
# a resolvable persistent identifier rather than an invented one.
|
|
160
|
+
lang = entities[workflow["programmingLanguage"]["@id"]]
|
|
161
|
+
assert lang["@type"] == "ComputerLanguage"
|
|
162
|
+
assert lang["name"] == "Bilayers"
|
|
163
|
+
assert lang["@id"] == "https://doi.org/10.5281/zenodo.17652333"
|
|
164
|
+
|
|
165
|
+
def test_payload_files_are_copied_into_crate(self, tmp_path, bilayers_config, ome_zarr_dirs):
|
|
166
|
+
"""Every entity in the crate is backed by a file that is actually there."""
|
|
167
|
+
input_zarr, output_zarr = ome_zarr_dirs
|
|
168
|
+
dest = tmp_path / "crate"
|
|
169
|
+
dest.mkdir()
|
|
170
|
+
|
|
171
|
+
create_ro_crate(
|
|
172
|
+
dest_path=str(dest),
|
|
173
|
+
workflow_schema_filename=str(bilayers_config),
|
|
174
|
+
input_path=str(input_zarr),
|
|
175
|
+
output_paths=[str(output_zarr)],
|
|
176
|
+
)
|
|
177
|
+
|
|
178
|
+
assert (dest / "config.yaml").read_text() == BILAYERS_CONFIG
|
|
179
|
+
# The zarr directories are copied whole, nested chunks included.
|
|
180
|
+
assert (dest / "input.ome.zarr" / ".zattrs").exists()
|
|
181
|
+
assert (dest / "input.ome.zarr" / "0" / "0.0.0").read_bytes() == b"\x01\x02\x03"
|
|
182
|
+
assert (dest / "output.ome.zarr" / ".zattrs").exists()
|
|
183
|
+
|
|
184
|
+
# No entity may describe a file that is missing from the crate.
|
|
185
|
+
for eid in self._read_ids(dest):
|
|
186
|
+
if eid.startswith("#") or eid.startswith("http") or eid == "./":
|
|
187
|
+
continue
|
|
188
|
+
assert (dest / eid).exists(), f"entity '{eid}' has no file in the crate"
|
|
189
|
+
|
|
190
|
+
def test_multiple_outputs_get_distinct_parameters(self, tmp_path, bilayers_config):
|
|
191
|
+
"""Each output dataset gets its own FormalParameter."""
|
|
192
|
+
input_zarr = tmp_path / "input.ome.zarr"
|
|
193
|
+
input_zarr.mkdir()
|
|
194
|
+
outputs = []
|
|
195
|
+
for name in ("a.ome.zarr", "b.ome.zarr"):
|
|
196
|
+
out = tmp_path / name
|
|
197
|
+
out.mkdir()
|
|
198
|
+
outputs.append(str(out))
|
|
199
|
+
dest = tmp_path / "crate"
|
|
200
|
+
dest.mkdir()
|
|
201
|
+
|
|
202
|
+
create_ro_crate(
|
|
203
|
+
dest_path=str(dest),
|
|
204
|
+
workflow_schema_filename=str(bilayers_config),
|
|
205
|
+
input_path=str(input_zarr),
|
|
206
|
+
output_paths=outputs,
|
|
207
|
+
)
|
|
208
|
+
|
|
209
|
+
entities = self._read_entities(dest)
|
|
210
|
+
output_ids = [ref["@id"] for ref in entities["config.yaml"]["output"]]
|
|
211
|
+
assert output_ids == ["#output", "#output_2"]
|
|
212
|
+
assert entities["a.ome.zarr/"]["exampleOfWork"] == {"@id": "#output"}
|
|
213
|
+
assert entities["b.ome.zarr/"]["exampleOfWork"] == {"@id": "#output_2"}
|
|
214
|
+
|
|
215
|
+
@staticmethod
|
|
216
|
+
def _read_entities(dest: Path) -> dict:
|
|
217
|
+
metadata = json.loads((dest / "ro-crate-metadata.json").read_text())
|
|
218
|
+
return {entity["@id"]: entity for entity in metadata["@graph"]}
|
|
219
|
+
|
|
220
|
+
@staticmethod
|
|
221
|
+
def _read_ids(dest: Path) -> set[str]:
|
|
222
|
+
metadata = json.loads((dest / "ro-crate-metadata.json").read_text())
|
|
223
|
+
return {entity["@id"] for entity in metadata["@graph"]}
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
if __name__ == "__main__":
|
|
227
|
+
import pprint
|
|
228
|
+
import tempfile
|
|
229
|
+
|
|
230
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
231
|
+
tmp_path = Path(tmp)
|
|
232
|
+
|
|
233
|
+
bilayers_config = make_bilayers_config(tmp_path)
|
|
234
|
+
input_zarr, output_zarr = make_ome_zarr_dirs(tmp_path)
|
|
235
|
+
dest = tmp_path / "crate"
|
|
236
|
+
dest.mkdir()
|
|
237
|
+
|
|
238
|
+
create_ro_crate(
|
|
239
|
+
dest_path=str(dest),
|
|
240
|
+
workflow_schema_filename=str(bilayers_config),
|
|
241
|
+
input_path=str(input_zarr),
|
|
242
|
+
output_paths=[str(output_zarr)],
|
|
243
|
+
)
|
|
244
|
+
|
|
245
|
+
metadata = json.loads((dest / "ro-crate-metadata.json").read_text())
|
|
246
|
+
pprint.pprint(metadata)
|