bfio 2.3.6__tar.gz → 2.4.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: bfio
3
- Version: 2.3.6
3
+ Version: 2.4.1
4
4
  Summary: Simple reading and writing classes for tiled tiffs using Bio-Formats.
5
5
  Author-email: Nick Schaub <nick.schaub@nih.gov>, Sameeul Bashir Samee <sameeul.samee@axleinfo.com>
6
6
  Maintainer-email: Sameeul Bashir Samee <sameeul.samee@axleinfo.com>
@@ -42,11 +42,12 @@ Requires-Dist: zarr>=2.6.1
42
42
  Requires-Dist: scyjava
43
43
  Requires-Dist: jpype1
44
44
  Requires-Dist: tifffile>=2022.8.12
45
+ Requires-Dist: bfiocpp
45
46
  Provides-Extra: dev
46
47
  Requires-Dist: ome_zarr; extra == "dev"
47
48
  Requires-Dist: requests>=2.26.0; extra == "dev"
48
49
 
49
- # **B**io**F**ormats **I**nput/**O**utput utility (bfio 2.3.6)
50
+ # **B**io**F**ormats **I**nput/**O**utput utility (bfio 2.4.1)
50
51
 
51
52
  [![Documentation Status](https://readthedocs.org/projects/bfio/badge/?version=latest)](https://bfio.readthedocs.io/en/latest/?badge=latest)
52
53
  [![PyPI](https://img.shields.io/pypi/v/bfio)](https://pypi.org/project/bfio/)
@@ -98,15 +99,15 @@ be installed using pip:
98
99
 
99
100
  ## Docker
100
101
 
101
- ### polusai/bfio:2.3.6
102
+ ### polusai/bfio:2.4.1
102
103
 
103
104
  Ubuntu based container with bfio and all dependencies (including Java).
104
105
 
105
- ### polusai/bfio:2.3.6-imagej
106
+ ### polusai/bfio:2.4.1-imagej
106
107
 
107
108
  Same as above, except comes with ImageJ and PyImageJ.
108
109
 
109
- ### polusai/bfio:2.3.6-tensorflow
110
+ ### polusai/bfio:2.4.1-tensorflow
110
111
 
111
112
  Tensorflow container with bfio installed.
112
113
 
@@ -1,4 +1,4 @@
1
- # **B**io**F**ormats **I**nput/**O**utput utility (bfio 2.3.6)
1
+ # **B**io**F**ormats **I**nput/**O**utput utility (bfio 2.4.1)
2
2
 
3
3
  [![Documentation Status](https://readthedocs.org/projects/bfio/badge/?version=latest)](https://bfio.readthedocs.io/en/latest/?badge=latest)
4
4
  [![PyPI](https://img.shields.io/pypi/v/bfio)](https://pypi.org/project/bfio/)
@@ -50,15 +50,15 @@ be installed using pip:
50
50
 
51
51
  ## Docker
52
52
 
53
- ### polusai/bfio:2.3.6
53
+ ### polusai/bfio:2.4.1
54
54
 
55
55
  Ubuntu based container with bfio and all dependencies (including Java).
56
56
 
57
- ### polusai/bfio:2.3.6-imagej
57
+ ### polusai/bfio:2.4.1-imagej
58
58
 
59
59
  Same as above, except comes with ImageJ and PyImageJ.
60
60
 
61
- ### polusai/bfio:2.3.6-tensorflow
61
+ ### polusai/bfio:2.4.1-tensorflow
62
62
 
63
63
  Tensorflow container with bfio installed.
64
64
 
@@ -10,7 +10,8 @@ requires = [
10
10
  "zarr>=2.6.1",
11
11
  "scyjava",
12
12
  "jpype1",
13
- "tifffile>=2022.8.12"
13
+ "tifffile>=2022.8.12",
14
+ "bfiocpp"
14
15
  ]
15
16
  build-backend = "setuptools.build_meta"
16
17
 
@@ -25,7 +26,8 @@ dependencies = [
25
26
  "zarr>=2.6.1",
26
27
  "scyjava",
27
28
  "jpype1",
28
- "tifffile>=2022.8.12"
29
+ "tifffile>=2022.8.12",
30
+ "bfiocpp"
29
31
  ]
30
32
 
31
33
  description = "Simple reading and writing classes for tiled tiffs using Bio-Formats."
@@ -84,7 +86,7 @@ namespaces = false
84
86
 
85
87
 
86
88
  [tool.bumpversion]
87
- current_version = "2.3.6"
89
+ current_version = "2.4.1"
88
90
  commit = true
89
91
  tag = true
90
92
  commit_args = "--no-verify"
@@ -0,0 +1 @@
1
+ 2.4.1
@@ -26,4 +26,5 @@ except FileNotFoundError:
26
26
  logger.info("VERSION = {}".format(__version__))
27
27
 
28
28
 
29
- from .bfio import BioReader, BioWriter, start # NOQA: F401, E402
29
+ from .bfio import BioReader, BioWriter # NOQA: F401, E402
30
+ from .utils import start # NOQA: F401, E402
@@ -1,27 +1,26 @@
1
1
  # -*- coding: utf-8 -*-
2
2
  # import core packages
3
- import copy
4
3
  import io
5
4
  import logging
6
5
  import shutil
7
6
  import struct
8
7
  from concurrent.futures import ThreadPoolExecutor, as_completed
9
8
  from pathlib import Path
10
- from typing import Dict, Optional, List, Tuple
9
+ from typing import Dict, List, Tuple
11
10
  import threading
12
11
 
13
12
  # Third party packages
14
13
  import imagecodecs
15
14
  import numpy
16
15
  import ome_types
17
- import re
18
16
  from tifffile import tifffile
19
17
  from xml.etree import ElementTree as ET
20
- from xsdata.utils.dates import DateTimeParser
18
+
21
19
 
22
20
  # bfio internals
23
- import bfio
21
+ from bfio import __version__ as version
24
22
  import bfio.base_classes
23
+ from bfio.utils import start, clean_ome_xml_for_known_issues
25
24
 
26
25
  logging.basicConfig(
27
26
  format="%(asctime)s - %(name)-8s - %(levelname)-8s - %(message)s",
@@ -29,329 +28,6 @@ logging.basicConfig(
29
28
  )
30
29
  logger = logging.getLogger("bfio.backends")
31
30
 
32
- KNOWN_INVALID_OME_XSD_REFERENCES = [
33
- "www.openmicroscopy.org/Schemas/ome/2013-06",
34
- "www.openmicroscopy.org/Schemas/OME/2012-03",
35
- "www.openmicroscopy.org/Schemas/sa/2013-06s",
36
- ]
37
- REPLACEMENT_OME_XSD_REFERENCE = "www.openmicroscopy.org/Schemas/OME/2016-06"
38
-
39
- ome_formats = {
40
- "detector_id": lambda x: f"Detector:{x}",
41
- "instrument_id": lambda x: f"Instrument:{x}",
42
- "image_id": lambda x: f"Image:{x}",
43
- }
44
-
45
-
46
- def clean_ome_xml_for_known_issues(xml: str) -> str:
47
- """Clean an OME XML string.
48
-
49
- This was modified from from AICSImageIO:
50
- https://github.com/AllenCellModeling/aicsimageio/blob/240c1c76a7e884aa37e11a1fbe0fcbb89fea6515/aicsimageio/metadata/utils.py#L187
51
- """
52
- # Store list of changes to print out with warning
53
- metadata_changes = []
54
-
55
- # Fix xsd reference
56
- # This is from OMEXML object just having invalid reference
57
- for known_invalid_ref in KNOWN_INVALID_OME_XSD_REFERENCES:
58
- if known_invalid_ref in xml:
59
- xml = xml.replace(
60
- known_invalid_ref,
61
- REPLACEMENT_OME_XSD_REFERENCE,
62
- )
63
- metadata_changes.append(
64
- f"Replaced '{known_invalid_ref}' with "
65
- f"'{REPLACEMENT_OME_XSD_REFERENCE}'."
66
- )
67
-
68
- # Read in XML
69
- try:
70
- root = ET.fromstring(xml)
71
- except ET.ParseError:
72
- # remove special char if initial parsing fails
73
- xml = xml.replace("&#0;", "")
74
- root = ET.fromstring(xml)
75
-
76
- # Get the namespace
77
- # In XML etree this looks like
78
- # "{http://www.openmicroscopy.org/Schemas/OME/2016-06}"
79
- # and must prepend any etree finds
80
- namespace_matches = re.match(r"\{.*\}", root.tag)
81
- if namespace_matches is not None:
82
- namespace = namespace_matches.group(0)
83
- else:
84
- raise ValueError("XML does not contain a namespace")
85
-
86
- # Fix MicroManager Instrument and Detector
87
- instrument = root.find(f"{namespace}Instrument")
88
- if instrument is not None:
89
- instrument_id = instrument.get("ID")
90
- if instrument_id == "Microscope":
91
- ome_instrument_id = ome_formats["instrument_id"](0)
92
- instrument.set("ID", ome_instrument_id)
93
- metadata_changes.append(
94
- f"Updated attribute 'ID' from '{instrument_id}' to "
95
- f"'{ome_instrument_id}' on Instrument element."
96
- )
97
-
98
- for detector_index, detector in enumerate(
99
- instrument.findall(f"{namespace}Detector")
100
- ):
101
- detector_id = detector.get("ID")
102
- if detector_id is not None:
103
- # Create ome detector id if needed
104
- ome_detector_id = None
105
- if detector_id == "Camera":
106
- ome_detector_id = ome_formats["detector_id"](detector_index)
107
- elif not detector_id.startswith("Detector:"):
108
- ome_detector_id = ome_formats["detector_id"](detector_id)
109
-
110
- # Apply ome detector id if replaced
111
- if ome_detector_id is not None:
112
- detector.set("ID", ome_detector_id)
113
- metadata_changes.append(
114
- f"Updated attribute 'ID' from '{detector_id}' to "
115
- f"'{ome_detector_id}' on Detector element at "
116
- f"position {detector_index}."
117
- )
118
-
119
- # Find all Image elements and fix IDs and refs to fixed instruments
120
- # This is for certain for test files of o.urs and ACTK files
121
- for image_index, image in enumerate(root.findall(f"{namespace}Image")):
122
- image_id = image.get("ID")
123
- if image_id is not None:
124
- found_image_id = image_id
125
-
126
- if not found_image_id.startswith("Image"):
127
- ome_image_id = ome_formats["image_id"](found_image_id)
128
- image.set("ID", ome_image_id)
129
- metadata_changes.append(
130
- f"Updated attribute 'ID' from '{image_id}' to '{ome_image_id}' "
131
- f"on Image element at position {image_index}."
132
- )
133
- # Fix bad acquisition date refs
134
- acq_date_ref = image.find(f"{namespace}AcquisitionDate")
135
- if acq_date_ref is not None:
136
- try:
137
- parser = DateTimeParser(acq_date_ref.text, "%Y-%m-%dT%H:%M:%S%z")
138
- next(parser.parse())
139
- except ValueError:
140
- image.remove(acq_date_ref)
141
- metadata_changes.append("Removed badly formatted AcquisitionDate.")
142
-
143
- # Fix MicroManager bad instrument refs
144
- instrument_ref = image.find(f"{namespace}InstrumentRef")
145
- if instrument_ref is not None:
146
- instrument_ref_id = instrument_ref.get("ID")
147
- if instrument_ref_id == "Microscope":
148
- instrument_ref.set("ID", ome_instrument_id)
149
-
150
- # Find all Pixels elements and fix IDs
151
- for pixels_index, pixels in enumerate(image.findall(f"{namespace}Pixels")):
152
- pixels_id = pixels.get("ID")
153
- if pixels_id is not None:
154
- found_pixels_id = pixels_id
155
-
156
- if not found_pixels_id.startswith("Pixels"):
157
- pixels.set("ID", f"Pixels:{found_pixels_id}")
158
- metadata_changes.append(
159
- f"Updated attribute 'ID' from '{found_pixels_id}' to "
160
- f"Pixels:{found_pixels_id}' on Pixels element at "
161
- f"position {pixels_index}."
162
- )
163
-
164
- # Determine if there is an out-of-order channel / plane elem
165
- # This is due to OMEXML "add channel" function
166
- # That added Channels and appropriate Planes to the XML
167
- # But, placed them in:
168
- # Channel
169
- # Plane
170
- # Plane
171
- # ...
172
- # Channel
173
- # Plane
174
- # Plane
175
- #
176
- # Instead of grouped together:
177
- # Channel
178
- # Channel
179
- # ...
180
- # Plane
181
- # Plane
182
- # ...
183
- #
184
- # This effects all CFE files (new and old) but for different reasons
185
- pixels_children_out_of_order = False
186
- encountered_something_besides_channel = False
187
- encountered_plane = False
188
- for child in pixels:
189
- if child.tag != f"{namespace}Channel":
190
- encountered_something_besides_channel = True
191
- if child.tag == f"{namespace}Plane":
192
- encountered_plane = True
193
- if (
194
- encountered_something_besides_channel
195
- and child.tag == f"{namespace}Channel"
196
- ):
197
- pixels_children_out_of_order = True
198
- break
199
- if encountered_plane and child.tag in [
200
- f"{namespace}{t}" for t in ["BinData", "TiffData", "MetadataOnly"]
201
- ]:
202
- pixels_children_out_of_order = True
203
- break
204
-
205
- # Ensure order of:
206
- # channels -> bindata | tiffdata | metadataonly -> planes
207
- if pixels_children_out_of_order:
208
- # Get all relevant elems
209
- channels = [
210
- copy.deepcopy(c) for c in pixels.findall(f"{namespace}Channel")
211
- ]
212
- bin_data = [
213
- copy.deepcopy(b) for b in pixels.findall(f"{namespace}BinData")
214
- ]
215
- tiff_data = [
216
- copy.deepcopy(t) for t in pixels.findall(f"{namespace}TiffData")
217
- ]
218
- # There should only be one metadata only element but to standardize
219
- # list comprehensions later we findall
220
- metadata_only = [
221
- copy.deepcopy(m) for m in pixels.findall(f"{namespace}MetadataOnly")
222
- ]
223
- planes = [copy.deepcopy(p) for p in pixels.findall(f"{namespace}Plane")]
224
-
225
- # Old (2018 ish) cell feature explorer files sometimes contain both
226
- # an empty metadata only element and filled tiffdata elements
227
- # Since the metadata only elements are empty we can check this and
228
- # choose the tiff data elements instead
229
- #
230
- # First check if there are any metadata only elements
231
- if len(metadata_only) == 1:
232
- # Now check if _one of_ of the other two choices are filled
233
- # ^ in Python is XOR
234
- if (len(bin_data) > 0) ^ (len(tiff_data) > 0):
235
- metadata_children = list(metadata_only[0])
236
- # Now check if the metadata only elem has no children
237
- if len(metadata_children) == 0:
238
- # If so, just "purge" by creating empty list
239
- metadata_only = []
240
-
241
- # If there are children elements
242
- # Return XML and let XMLSchema Validation show error
243
- else:
244
- return xml
245
-
246
- # After cleaning metadata only, validate the normal behaviors of
247
- # OME schema
248
- #
249
- # Validate that there is only one of bindata, tiffdata, or metadata
250
- if len(bin_data) > 0:
251
- if len(tiff_data) == 0 and len(metadata_only) == 0:
252
- selected_choice = bin_data
253
- else:
254
- # Return XML and let XMLSchema Validation show error
255
- return xml
256
- elif len(tiff_data) > 0:
257
- if len(bin_data) == 0 and len(metadata_only) == 0:
258
- selected_choice = tiff_data
259
- else:
260
- # Return XML and let XMLSchema Validation show error
261
- return xml
262
- elif len(metadata_only) == 1:
263
- if len(bin_data) == 0 and len(tiff_data) == 0:
264
- selected_choice = metadata_only
265
- else:
266
- # Return XML and let XMLSchema Validation show error
267
- return xml
268
- else:
269
- # Return XML and let XMLSchema Validation show error
270
- return xml
271
-
272
- # Remove all children from element to be replaced
273
- # with ordered elements
274
- for elem in list(pixels):
275
- pixels.remove(elem)
276
-
277
- # Re-attach elements
278
- for channel in channels:
279
- pixels.append(channel)
280
- for elem in selected_choice:
281
- pixels.append(elem)
282
- for plane in planes:
283
- pixels.append(plane)
284
-
285
- metadata_changes.append(
286
- f"Reordered children of Pixels element at "
287
- f"position {pixels_index}."
288
- )
289
-
290
- # This is a result of dumping basically all experiment metadata
291
- # into "StructuredAnnotation" blocks
292
- #
293
- # This affects new (2020) Cell Feature Explorer files
294
- #
295
- # Because these are structured annotations we don't want to mess with anyones
296
- # besides the AICS generated bad structured annotations
297
- aics_anno_removed_count = 0
298
- sa = root.find(f"{namespace}StructuredAnnotations")
299
- if sa is not None:
300
- for xml_anno in sa.findall(f"{namespace}XMLAnnotation"):
301
- # At least these are namespaced
302
- if xml_anno.get("Namespace") == "alleninstitute.org/CZIMetadata":
303
- # Get ID because some elements have annotation refs
304
- # in both the base Image element and all plane elements
305
- aics_anno_id = xml_anno.get("ID")
306
- for image in root.findall(f"{namespace}Image"):
307
- for anno_ref in image.findall(f"{namespace}AnnotationRef"):
308
- if anno_ref.get("ID") == aics_anno_id:
309
- image.remove(anno_ref)
310
-
311
- # Clean planes
312
- if image is not None:
313
- found_image = image
314
-
315
- pixels_planes: Optional[ET.Element] = found_image.find(
316
- f"{namespace}Pixels"
317
- )
318
- if pixels_planes is not None:
319
- for plane in pixels_planes.findall(f"{namespace}Plane"):
320
- for anno_ref in plane.findall(
321
- f"{namespace}AnnotationRef"
322
- ):
323
- if anno_ref.get("ID") == aics_anno_id:
324
- plane.remove(anno_ref)
325
-
326
- # Remove the whole etree
327
- sa.remove(xml_anno)
328
- aics_anno_removed_count += 1
329
-
330
- # Log changes
331
- if aics_anno_removed_count > 0:
332
- metadata_changes.append(
333
- f"Removed {aics_anno_removed_count} AICS generated XMLAnnotations."
334
- )
335
-
336
- # If there are no annotations in StructuredAnnotations, remove it
337
- if sa is not None:
338
- if len(list(sa)) == 0:
339
- root.remove(sa)
340
-
341
- # If any piece of metadata was changed alert and rewrite
342
- if len(metadata_changes) > 0:
343
- # Register namespace
344
- ET.register_namespace("", f"http://{REPLACEMENT_OME_XSD_REFERENCE}")
345
-
346
- # Write out cleaned XML to string
347
- xml = ET.tostring(
348
- root,
349
- encoding="unicode",
350
- method="xml",
351
- )
352
-
353
- return xml
354
-
355
31
 
356
32
  class PythonReader(bfio.base_classes.AbstractReader):
357
33
  logger = logging.getLogger("bfio.backends.PythonReader")
@@ -596,8 +272,8 @@ class PythonReader(bfio.base_classes.AbstractReader):
596
272
 
597
273
  self.logger.debug("read_image(): _tile_indices = {}".format(self._tile_indices))
598
274
 
599
- if self.frontend.max_workers > 1:
600
- with ThreadPoolExecutor(self.frontend.max_workers) as executor:
275
+ if self.frontend._max_workers > 1:
276
+ with ThreadPoolExecutor(self.frontend._max_workers) as executor:
601
277
  # cast to list so that any read errors are raised
602
278
  list(
603
279
  executor.map(
@@ -958,7 +634,7 @@ class PythonWriter(bfio.base_classes.AbstractWriter):
958
634
 
959
635
  offsetsize = self._writer.tiff.offsetsize
960
636
 
961
- self._compresstag = tifffile.TIFF.COMPRESSION.ADOBE_DEFLATE
637
+ self._compresstag = tifffile.COMPRESSION.ADOBE_DEFLATE
962
638
 
963
639
  # normalize data shape to 5D or 6D, depending on volume:
964
640
  # (pages, planar_samples, height, width, contig_samples)
@@ -982,7 +658,7 @@ class PythonWriter(bfio.base_classes.AbstractWriter):
982
658
  description = ome_types.to_xml(self.frontend._metadata)
983
659
 
984
660
  self._addtag(270, "s", 0, description, writeonce=True) # Description
985
- self._addtag(305, "s", 0, f"bfio v{bfio.__version__}") # Software
661
+ self._addtag(305, "s", 0, f"bfio v{version}") # Software
986
662
  # addtag(306, 's', 0, datetime, writeonce=True)
987
663
  self._addtag(259, "H", 1, self._compresstag) # Compression
988
664
  self._addtag(256, "I", 1, self._datashape[-2]) # ImageWidth
@@ -999,7 +675,7 @@ class PythonWriter(bfio.base_classes.AbstractWriter):
999
675
  self._addtag(258, "H", 1, self._bitspersample)
1000
676
 
1001
677
  # PhotometricInterpretation
1002
- self._addtag(262, "H", 1, tifffile.TIFF.PHOTOMETRIC.MINISBLACK.value)
678
+ self._addtag(262, "H", 1, tifffile.PHOTOMETRIC.MINISBLACK.value)
1003
679
 
1004
680
  if self.frontend.physical_size_x[0] is not None:
1005
681
  self._addtag(
@@ -1155,8 +831,8 @@ class PythonWriter(bfio.base_classes.AbstractWriter):
1155
831
  def compress(page_index, tile_index, data, level=1):
1156
832
  return (page_index, tile_index, imagecodecs.deflate_encode(data, level))
1157
833
 
1158
- if self.frontend.max_workers > 1:
1159
- with ThreadPoolExecutor(max_workers=self.frontend.max_workers) as executor:
834
+ if self.frontend._max_workers > 1:
835
+ with ThreadPoolExecutor(max_workers=self.frontend._max_workers) as executor:
1160
836
  compressed_tiles = []
1161
837
  for page_index, tileiter in tileiters:
1162
838
  for tileindex, tile in zip(tiles, tileiter):
@@ -1212,7 +888,7 @@ try:
1212
888
 
1213
889
  def _load_java_classes(self):
1214
890
  if not jpype.isJVMStarted():
1215
- bfio.start()
891
+ start()
1216
892
 
1217
893
  global ImageReader
1218
894
  from loci.formats import ImageReader
@@ -1353,7 +1029,7 @@ try:
1353
1029
 
1354
1030
  def _load_java_classes(self):
1355
1031
  if not jpype.isJVMStarted():
1356
- bfio.start()
1032
+ start()
1357
1033
 
1358
1034
  global OMETiffWriter
1359
1035
  from loci.formats.out import OMETiffWriter
@@ -1701,8 +1377,8 @@ try:
1701
1377
  ] = data
1702
1378
 
1703
1379
  def _read_image(self, X, Y, Z, C, T, output):
1704
- if self.frontend.max_workers > 1:
1705
- with ThreadPoolExecutor(self.frontend.max_workers) as executor:
1380
+ if self.frontend._max_workers > 1:
1381
+ with ThreadPoolExecutor(self.frontend._max_workers) as executor:
1706
1382
  executor.map(self._process_chunk, self._tile_indices)
1707
1383
  else:
1708
1384
  for args in self._tile_indices:
@@ -1803,8 +1479,8 @@ try:
1803
1479
  )
1804
1480
 
1805
1481
  def _write_image(self, X, Y, Z, C, T, image):
1806
- if self.frontend.max_workers > 1:
1807
- with ThreadPoolExecutor(self.frontend.max_workers) as executor:
1482
+ if self.frontend._max_workers > 1:
1483
+ with ThreadPoolExecutor(self.frontend._max_workers) as executor:
1808
1484
  executor.map(self._process_chunk, self._tile_indices)
1809
1485
  else:
1810
1486
  for args in self._tile_indices: