setiastrosuitepro 1.6.5.post3__py3-none-any.whl

setiastro/saspro/xisf.py

@@ -0,0 +1,1213 @@
+# coding: utf-8
+
+"""
+XISF Encoder/Decoder (see https://pixinsight.com/xisf/).
+
+This implementation is not endorsed nor related with PixInsight development team.
+
+Copyright (C) 2021-2023 Sergio Díaz, sergiodiaz.eu
+
+This program is free software: you can redistribute it and/or modify it
+under the terms of the GNU General Public License as published by the
+Free Software Foundation, version 3 of the License.
+
+This program is distributed in the hope that it will be useful, but WITHOUT
+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+more details.
+
+You should have received a copy of the GNU General Public License along with
+this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+from importlib.metadata import version
+
+
+import platform
+import xml.etree.ElementTree as ET
+import numpy as np
+import lz4.block  # https://python-lz4.readthedocs.io/en/stable/lz4.block.html
+import zlib  # https://docs.python.org/3/library/zlib.html
+import zstandard  # https://python-zstandard.readthedocs.io/en/stable/
+import base64
+import sys
+from datetime import datetime
+import ast
+
+__version__ = "1.0.1"
+
+def _is_attached_or_inline_property(p_dict):
+    return "location" in p_dict  # location implies inline/embedded/attachment
+
+def _make_lazy(p_dict):
+    p_dict["_lazy"] = True
+    return p_dict
+
+class XISF:
+    """Implements an baseline XISF Decoder and a simple baseline Encoder.
+    It parses metadata from Image and Metadata XISF core elements. Image data is returned as a numpy ndarray
+    (using the "channels-last" convention by default).
+
+    What's supported:
+    - Monolithic XISF files only
+        - XISF data blocks with attachment, inline or embedded block locations
+        - Planar pixel storage models, *however it assumes 2D images only* (with multiple channels)
+        - UInt8/16/32 and Float32/64 pixel sample formats
+        - Grayscale and RGB color spaces
+    - Decoding:
+        - multiple Image core elements from a monolithic XISF file
+        - Support all standard compression codecs defined in this specification for decompression
+          (zlib/lz4[hc]/zstd + byte shuffling)
+    - Encoding:
+        - Single image core element with an attached data block
+        - Support all standard compression codecs defined in this specification for decompression
+          (zlib/lz4[hc]/zstd + byte shuffling)
+    - "Atomic" properties (scalar types, String, TimePoint), Vector and Matrix (e.g. astrometric
+      solutions)
+    - Metadata and FITSKeyword core elements
+
+    What's not supported (at least by now):
+    - Read pixel data in the normal pixel storage models
+    - Read pixel data in the planar pixel storage models other than 2D images
+    - Complex and Table properties
+    - Any other not explicitly supported core elements (Resolution, Thumbnail, ICCProfile, etc.)
+
+    Usage example:
+    ```
+    from setiastro.saspro.xisf import XISF
+    import matplotlib.pyplot as plt
+    xisf = XISF("file.xisf")
+    file_meta = xisf.get_file_metadata()
+    file_meta
+    ims_meta = xisf.get_images_metadata()
+    ims_meta
+    im_data = xisf.read_image(0)
+    plt.imshow(im_data)
+    plt.show()
+    XISF.write(
+        "output.xisf", im_data,
+        creator_app="My script v1.0", image_metadata=ims_meta[0], xisf_metadata=file_meta,
+        codec='lz4hc', shuffle=True
+    )
+    ```
+
+    If the file is not huge and it contains only an image (or you're interested just in one of the
+    images inside the file), there is a convenience method for reading the data and the metadata:
+    ```
+    from setiastro.saspro.xisf import XISF
+    import matplotlib.pyplot as plt
+    im_data = XISF.read("file.xisf")
+    plt.imshow(im_data)
+    plt.show()
+    ```
+
+    The XISF format specification is available at https://pixinsight.com/doc/docs/XISF-1.0-spec/XISF-1.0-spec.html
+    """
+
+    # Static attributes
+    _creator_app = f"Python {platform.python_version()}"
+    _creator_module = f"XISF Python Module v{__version__} github.com/sergio-dr/xisf"
+    _signature = b"XISF0100"  # Monolithic
+    _headerlength_len = 4
+    _reserved_len = 4
+    _xml_ns = {"xisf": "http://www.pixinsight.com/xisf"}
+    _xisf_attrs = {
+        "xmlns": "http://www.pixinsight.com/xisf",
+        "xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance",
+        "version": "1.0",
+        "xsi:schemaLocation": "http://www.pixinsight.com/xisf http://pixinsight.com/xisf/xisf-1.0.xsd",
+    }
+    _compression_def_level = {
+        "zlib": 6,  # 1..9, default: 6 as indicated in https://docs.python.org/3/library/zlib.html
+        "lz4": 0,  # no other values, as indicated in https://python-lz4.readthedocs.io/en/stable/lz4.block.html
+        "lz4hc": 9,  # 1..12, (4-9 recommended), default: 9 as indicated in https://python-lz4.readthedocs.io/en/stable/lz4.block.html
+        "zstd": 3,  # 1..22, (3-9 recommended), default: 3 as indicated in https://facebook.github.io/zstd/zstd_manual.html
+    }
+    _block_alignment_size = 4096
+    _max_inline_block_size = 3072
+
+    def __init__(self, fname):
+        """Opens a XISF file and extract its metadata. To get the metadata and the images, see get_file_metadata(),
+        get_images_metadata() and read_image().
+        Args:
+            fname: filename
+
+        Returns:
+            XISF object.
+        """
+        self._fname = fname
+        self._headerlength = None
+        self._xisf_header = None
+        self._xisf_header_xml = None
+        self._images_meta = None
+        self._file_meta = None
+        ET.register_namespace("", self._xml_ns["xisf"])
+
+        self._read()
+
+    def _read(self):
+        with open(self._fname, "rb") as f:
+            # Check XISF signature
+            signature = f.read(len(self._signature))
+            if signature != self._signature:
+                raise ValueError("File doesn't have XISF signature")
+
+            # Get header length
+            self._headerlength = int.from_bytes(f.read(self._headerlength_len), byteorder="little")
+            # Equivalent:
+            # self._headerlength = np.fromfile(f, dtype=np.uint32, count=1)[0]
+
+            # Skip reserved field
+            _ = f.read(self._reserved_len)
+
+            # Get XISF (XML) Header
+            self._xisf_header = f.read(self._headerlength)
+            self._xisf_header_xml = ET.fromstring(self._xisf_header)
+        self._analyze_header()
+
+    def _analyze_header(self):
+        # Analyze header to get Data Blocks position and length
+        self._images_meta = []
+        for image in self._xisf_header_xml.findall("xisf:Image", self._xml_ns):
+            image_basic_meta = image.attrib
+
+            # Parse and replace geometry and location with tuples,
+            # parses and translates sampleFormat to numpy dtypes,
+            # and extend with metadata from children entities (FITSKeywords, XISFProperties)
+
+            #   The same FITS keyword can appear multiple times, so we have to
+            #   prepare a dict of lists. Each element in the list is a dict
+            #   that hold the value and the comment associated with the keyword.
+            #   Not as clear as I would like.
+            fits_keywords = {}
+            for a in image.findall("xisf:FITSKeyword", self._xml_ns):
+                fits_keywords.setdefault(a.attrib["name"], []).append(
+                    {
+                        "value": a.attrib["value"].strip("'").strip(" "),
+                        "comment": a.attrib["comment"],
+                    }
+                )
+
+            image_extended_meta = {
+                "geometry": self._parse_geometry(image.attrib["geometry"]),
+                "location": self._parse_location(image.attrib["location"]),
+                "dtype": self._parse_sampleFormat(image.attrib["sampleFormat"]),
+                "FITSKeywords": fits_keywords,
+                "XISFProperties": {
+                    p.attrib["id"]: prop
+                    for p in image.findall("xisf:Property", self._xml_ns)
+                    if (prop := self._process_property(p))
+                },
+            }
+            # Also parses compression attribute if present, converting it to a tuple
+            if "compression" in image.attrib:
+                image_extended_meta["compression"] = self._parse_compression(
+                    image.attrib["compression"]
+                )
+
+            # Merge basic and extended metadata in a dict
+            image_meta = {**image_basic_meta, **image_extended_meta}
+
+            # Append the image metadata to the list
+            self._images_meta.append(image_meta)
+
+        # Analyze header for file metadata
+        self._file_meta = {}
+        for p in self._xisf_header_xml.find("xisf:Metadata", self._xml_ns):
+            self._file_meta[p.attrib["id"]] = self._process_property(p)
+
+        # Parse additional XISF core elements: Resolution, ICCProfile, Thumbnail
+        self._parse_resolution_elements()
+        self._parse_icc_profiles()
+        self._parse_thumbnails()
+
+    def _parse_resolution_elements(self):
+        """Parse Resolution core elements and attach to image metadata."""
+        for i, image in enumerate(self._xisf_header_xml.findall("xisf:Image", self._xml_ns)):
+            res_elem = image.find("xisf:Resolution", self._xml_ns)
+            if res_elem is not None:
+                try:
+                    res_data = {
+                        "horizontal": float(res_elem.attrib.get("horizontal", 72.0)),
+                        "vertical": float(res_elem.attrib.get("vertical", 72.0)),
+                        "unit": res_elem.attrib.get("unit", "inch"),  # "inch" or "cm"
+                    }
+                    if i < len(self._images_meta):
+                        self._images_meta[i]["Resolution"] = res_data
+                except (ValueError, KeyError):
+                    pass
+
+    def _parse_icc_profiles(self):
+        """Parse ICCProfile core elements and attach to image metadata."""
+        for i, image in enumerate(self._xisf_header_xml.findall("xisf:Image", self._xml_ns)):
+            icc_elem = image.find("xisf:ICCProfile", self._xml_ns)
+            if icc_elem is not None:
+                try:
+                    icc_data = {"present": True}
+                    if "location" in icc_elem.attrib:
+                        loc = self._parse_location(icc_elem.attrib["location"])
+                        icc_data["location"] = loc
+                        # Read ICC profile binary data
+                        if loc[0] == "attachment" and len(loc) >= 3:
+                            icc_data["size"] = loc[2]
+                    if i < len(self._images_meta):
+                        self._images_meta[i]["ICCProfile"] = icc_data
+                except (ValueError, KeyError):
+                    pass
+
+    def _parse_thumbnails(self):
+        """Parse Thumbnail core elements and attach to image metadata."""
+        for i, image in enumerate(self._xisf_header_xml.findall("xisf:Image", self._xml_ns)):
+            thumb_elem = image.find("xisf:Thumbnail", self._xml_ns)
+            if thumb_elem is not None:
+                try:
+                    thumb_data = {
+                        "present": True,
+                        "geometry": self._parse_geometry(thumb_elem.attrib.get("geometry", "0:0:0")),
+                    }
+                    if "location" in thumb_elem.attrib:
+                        thumb_data["location"] = self._parse_location(thumb_elem.attrib["location"])
+                    if "sampleFormat" in thumb_elem.attrib:
+                        thumb_data["dtype"] = self._parse_sampleFormat(thumb_elem.attrib["sampleFormat"])
+                    if "colorSpace" in thumb_elem.attrib:
+                        thumb_data["colorSpace"] = thumb_elem.attrib["colorSpace"]
+                    if i < len(self._images_meta):
+                        self._images_meta[i]["Thumbnail"] = thumb_data
+                except (ValueError, KeyError):
+                    pass
+
+    def get_images_metadata(self):
+        """Provides the metadata of all image blocks contained in the XISF File, extracted from
+        the header (<Image> core elements). To get the actual image data, see read_image().
+
+        It outputs a dictionary m_i for each image, with the following structure:
+        ```
+        m_i = {
+            'geometry': (width, height, channels), # only 2D images (with multiple channels) are supported
+            'location': (pos, size), # used internally in read_image()
+            'dtype': np.dtype('...'), # derived from sampleFormat argument
+            'compression': (codec, uncompressed_size, item_size), # optional
+            'key': 'value', # other <Image> attributes are simply copied
+            ...,
+            'FITSKeywords': { <fits_keyword>: fits_keyword_values_list, ... },
+            'XISFProperties': { <xisf_property_name>: property_dict, ... }
+        }
+
+        where:
+
+        fits_keyword_values_list = [ {'value': <value>, 'comment': <comment> }, ...]
+        property_dict = {'id': <xisf_property_name>, 'type': <xisf_type>, 'value': property_value, ...}
+        ```
+
+        Returns:
+            list [ m_0, m_1, ..., m_{n-1} ] where m_i is a dict as described above.
+
+        """
+        return self._images_meta
+
+    def get_file_metadata(self):
+        """Provides the metadata from the header of the XISF File (<Metadata> core elements).
+
+        Returns:
+            dictionary with one entry per property: { <xisf_property_name>: property_dict, ... }
+            where:
+            ```
+            property_dict = {'id': <xisf_property_name>, 'type': <xisf_type>, 'value': property_value, ...}
+            ```
+
+        """
+        return self._file_meta
+
+    def get_metadata_xml(self):
+        """Returns the complete XML header as a xml.etree.ElementTree.Element object.
+
+        Returns:
+            xml.etree.ElementTree.Element: complete XML XISF header
+        """
+        return self._xisf_header_xml
+
+    def _read_data_block(self, elem):
+        method = elem["location"][0]
+        if method == "inline":
+            return self._read_inline_data_block(elem)
+        elif method == "embedded":
+            return self._read_embedded_data_block(elem)
+        elif method == "attachment":
+            return self._read_attached_data_block(elem)
+        else:
+            raise NotImplementedError(f"Data block location type '{method}' not implemented: {elem}")
+
+    @staticmethod
+    def _read_inline_data_block(elem):
+        method, encoding = elem["location"]
+        assert method == "inline"
+        return XISF._decode_inline_or_embedded_data(encoding, elem["value"], elem)
+
+    @staticmethod
+    def _read_embedded_data_block(elem):
+        assert elem["location"][0] == "embedded"
+        data_elem = ET.fromstring(elem["value"])
+        encoding, data = data_elem.attrib["encoding"], data_elem.text
+        return XISF._decode_inline_or_embedded_data(encoding, data, elem)
+
+    @staticmethod
+    def _decode_inline_or_embedded_data(encoding, data, elem):
+        encodings = {"base64": base64.b64decode, "hex": base64.b16decode}
+        if encoding not in encodings:
+            raise NotImplementedError(
+                f"Data block encoding type '{encoding}' not implemented: {elem}"
+            )
+
+        data = encodings[encoding](data)
+        if "compression" in elem:
+            data = XISF._decompress(data, elem)
+
+        return data
+
+    def _read_attached_data_block(self, elem):
+        # Position and size of the Data Block containing the image data
+        method, pos, size = elem["location"]
+
+        assert method == "attachment"
+
+        with open(self._fname, "rb") as f:
+            f.seek(pos)
+            data = f.read(size)
+
+        if "compression" in elem:
+            data = XISF._decompress(data, elem)
+
+        return data
+
+    def read_image(self, n=0, data_format="channels_last"):
+        """Extracts an image from a XISF object.
+
+        Args:
+            n: index of the image to extract in the list returned by get_images_metadata()
+            data_format: channels axis can be 'channels_first' or 'channels_last' (as used in
+            keras/tensorflow, pyplot's imshow, etc.), 0 by default.
+
+        Returns:
+            Numpy ndarray with the image data, in the requested format (channels_first or channels_last).
+
+        """
+        try:
+            meta = self._images_meta[n]
+        except IndexError as e:
+            if self._xisf_header is None:
+                raise RuntimeError("No file loaded") from e
+            elif not self._images_meta:
+                raise ValueError("File does not contain image data") from e
+            else:
+                raise ValueError(
+                    f"Requested image #{n}, valid range is [0..{len(self._images_meta) - 1}]"
+                ) from e
+
+        try:
+            # Assumes *two*-dimensional images (chc=channel count)
+            w, h, chc = meta["geometry"]
+        except ValueError as e:
+            raise NotImplementedError(
+                f"Assumed 2D channels (width, height, channels), found {meta['geometry']} geometry"
+            )
+
+        data = self._read_data_block(meta)
+        im_data = np.frombuffer(data, dtype=meta["dtype"])
+        im_data = im_data.reshape((chc, h, w))
+        return np.transpose(im_data, (1, 2, 0)) if data_format == "channels_last" else im_data
+
+    @staticmethod
+    def read(fname, n=0, image_metadata={}, xisf_metadata={}):
+        """Convenience method for reading a file containing a single image.
+
+        Args:
+            fname (string): filename
+            n (int, optional): index of the image to extract (in the list returned by get_images_metadata()). Defaults to 0.
+            image_metadata (dict, optional): dictionary that will be updated with the metadata of the image.
+            xisf_metadata (dict, optional): dictionary that will be updated with the metadata of the file.
+
+        Returns:
+            [np.ndarray]: Numpy ndarray with the image data, in the requested format (channels_first or channels_last).
+        """
+        xisf = XISF(fname)
+        xisf_metadata.update(xisf.get_file_metadata())
+        image_metadata.update(xisf.get_images_metadata()[n])
+        return xisf.read_image(n)
+
+    # if 'colorSpace' is not specified, im_data.shape[2] dictates if colorSpace is 'Gray' or 'RGB'
+    # For float sample formats, bounds="0:1" is assumed
+    @staticmethod
+    def write(
+        fname,
+        im_data,
+        creator_app=None,
+        image_metadata=None,
+        xisf_metadata=None,
+        codec=None,
+        shuffle=False,
+        level=None,
+    ):
+        """Writes an image (numpy array) to a XISF file. Compression may be requested but it only
+        will be used if it actually reduces the data size.
+
+        Args:
+            fname: filename (will overwrite if existing)
+            im_data: numpy ndarray with the image data
+            creator_app: string for XISF:CreatorApplication file property (defaults to python version in None provided)
+            image_metadata: dict with the same structure described for m_i in get_images_metadata().
+              Only 'FITSKeywords' and 'XISFProperties' keys are actually written, the rest are derived from im_data.
+            xisf_metadata: file metadata, dict with the same structure returned by get_file_metadata()
+            codec: compression codec ('zlib', 'lz4', 'lz4hc' or 'zstd'), or None to disable compression
+            shuffle: whether to apply byte-shuffling before compression (ignored if codec is None). Recommended
+              for 'lz4' ,'lz4hc' and 'zstd' compression algorithms.
+            level: for zlib, 1..9 (default: 6); for lz4hc, 1..12 (default: 9); for zstd, 1..22 (default: 3).
+              Higher means more compression.
+        Returns:
+            bytes_written: the total number of bytes written into the output file.
+            codec: The codec actually used, i.e., None if compression did not reduce the data block size so
+            compression was not finally used.
+
+        """
+        if image_metadata is None:
+            image_metadata = {}
+
+        if xisf_metadata is None:
+            xisf_metadata = {}
+
+        # Data block alignment
+        blk_sz = xisf_metadata.get("XISF:BlockAlignmentSize", {"value": XISF._block_alignment_size})[
+            "value"
+        ]
+        # Maximum inline block size (larger will be attached instead)
+        max_inline_blk_sz = xisf_metadata.get(
+            "XISF:MaxInlineBlockSize", {"value": XISF._max_inline_block_size}
+        )["value"]
+
+        # Prepare basic image metadata
+        def _create_image_metadata(im_data, id):
+            image_attrs = {"id": id}
+            if im_data.shape[2] == 3 or im_data.shape[2] == 1:
+                data_format = "channels_last"
+                geometry = (im_data.shape[1], im_data.shape[0], im_data.shape[2])
+                channels = im_data.shape[2]
+            else:
+                data_format = "channels_first"
+                geometry = im_data.shape
+                channels = im_data.shape[0]
+            image_attrs["geometry"] = "%d:%d:%d" % geometry
+            image_attrs["colorSpace"] = "Gray" if channels == 1 else "RGB"
+            image_attrs["sampleFormat"] = XISF._get_sampleFormat(im_data.dtype)
+            if image_attrs["sampleFormat"].startswith("Float"):
+                image_attrs["bounds"] = "0:1"  # Assumed
+            if sys.byteorder == "big" and image_attrs["sampleFormat"] != "UInt8":
+                image_attrs["byteOrder"] = "big"
+            return image_attrs, data_format
+
+        # Rearrange ndarray for data_format and serialize to bytes
+        def _prepare_image_data_block(im_data, data_format):
+            return np.transpose(im_data, (2, 0, 1)) if data_format == "channels_last" else im_data
+
+        # Serialize a data block, with optional compression (i.e., when codec is not None)
+        # Compression will be only applied if effectively reduces size
+        def _serialize_data_block(data, attr_dict, codec, level, shuffle):
+            data_block = data.tobytes()
+            uncompressed_size = data.nbytes
+            codec_str = codec
+
+            if codec is None:
+                data_size = uncompressed_size
+            else:
+                compressed_block = XISF._compress(data_block, codec, level, shuffle, data.itemsize)
+                compressed_size = len(compressed_block)
+
+                if compressed_size < uncompressed_size:
+                    # The ideal situation, compressing actually reduces size
+                    data_block, data_size = compressed_block, compressed_size
+
+                    # Add 'compression' image attribute: (codec:uncompressed-size[:item-size])
+                    if shuffle:
+                        codec_str += "+sh"
+                        attr_dict["compression"] = f"{codec_str}:{uncompressed_size}:{data.itemsize}"
+                    else:
+                        attr_dict["compression"] = f"{codec}:{uncompressed_size}"
+                else:
+                    # If there's no gain in compressing, just discard the compressed block
+                    # See https://pixinsight.com/forum.old/index.php?topic=10942.msg68043#msg68043
+                    # (In fact, PixInsight will show garbage image data if the data block is
+                    # compressed but the uncompressed size is smaller)
+                    data_size = uncompressed_size
+                    codec_str = None
+
+            return data_block, data_size, codec_str
+
+        # Overwrites/creates XISF metadata
+        def _update_xisf_metadata(creator_app, blk_sz, max_inline_blk_sz, codec, level):
+            # Create file metadata
+            xisf_metadata["XISF:CreationTime"] = {
+                "id": "XISF:CreationTime",
+                "type": "String",
+                "value": datetime.utcnow().isoformat(),
+            }
+            xisf_metadata["XISF:CreatorApplication"] = {
+                "id": "XISF:CreatorApplication",
+                "type": "String",
+                "value": creator_app if creator_app else XISF._creator_app,
+            }
+            xisf_metadata["XISF:CreatorModule"] = {
+                "id": "XISF:CreatorModule",
+                "type": "String",
+                "value": XISF._creator_module,
+            }
+            _OSes = {
+                "linux": "Linux",
+                "win32": "Windows",
+                "cygwin": "Windows",
+                "darwin": "macOS",
+            }
+            xisf_metadata["XISF:CreatorOS"] = {
+                "id": "XISF:CreatorOS",
+                "type": "String",
+                "value": _OSes[sys.platform],
+            }
+            xisf_metadata["XISF:BlockAlignmentSize"] = {
+                "id": "XISF:BlockAlignmentSize",
+                "type": "UInt16",
+                "value": blk_sz,
+            }
+            xisf_metadata["XISF:MaxInlineBlockSize"] = {
+                "id": "XISF:MaxInlineBlockSize",
+                "type": "UInt16",
+                "value": max_inline_blk_sz,
+            }
+            if codec is not None:
+                # Add XISF:CompressionCodecs and XISF:CompressionLevel to file metadata
+                xisf_metadata["XISF:CompressionCodecs"] = {
+                    "id": "XISF:CompressionCodecs",
+                    "type": "String",
+                    "value": codec,
+                }
+                xisf_metadata["XISF:CompressionLevel"] = {
+                    "id": "XISF:CompressionLevel",
+                    "type": "Int",
+                    "value": level if level else XISF._compression_def_level[codec],
+                }
+            else:
+                # Remove compression metadata if exists
+                try:
+                    del xisf_metadata["XISF:CompressionCodecs"]
+                    del xisf_metadata["XISF:CompressionLevel"]
+                except KeyError:
+                    pass  # Ignore if keys don't exist
+
+        def _compute_attached_positions(hdr_prov_sz, attached_blocks_locations):
+            # Computes aligned position nearest to the given one
+            _aligned_position = lambda pos: ((pos + blk_sz - 1) // blk_sz) * blk_sz
+
+            # Iterates data block positions until header size stabilizes
+            # (positions are represented as strings in the header so their
+            # values may impact header size, therefore changing data block
+            # positions in the file)
+            hdr_sz = hdr_prov_sz
+            prev_sum_len_positions = 0
+            while True:
+                # account for the size of the (provisional) header
+                pos = _aligned_position(hdr_sz)
+
+                # positions for data blocks of properties with attachment location
+                sum_len_positions = 0
+                for loc in attached_blocks_locations:
+                    # Save the (possibly provisional) position
+                    loc['position'] = pos
+                    # Accumulate the size of the position string
+                    sum_len_positions += len(str(pos))
+                    # Fast forward position adding the size, honoring alignment
+                    pos = _aligned_position(pos + loc['size'])
+
+                if sum_len_positions == prev_sum_len_positions:
+                    break
+
+                prev_sum_len_positions = sum_len_positions
+                hdr_sz = hdr_prov_sz + sum_len_positions
+
+            # Update data blocks positions in XML Header
+            for b in attached_blocks_locations:
+                xml_elem, pos, sz = b["xml"], b["position"], b["size"]
+                xml_elem.attrib["location"] = XISF._to_location(("attachment", pos, sz))
+
+        # Zero padding (used for reserved fields and data block alignment)
+        def _zero_pad(length):
+            assert length >= 0
+            return (0).to_bytes(length, byteorder="little")
+
+        # __/ Prepare image and its metadata \__________
+        im_id = image_metadata.get("id", "image")
+        im_attrs, data_format = _create_image_metadata(im_data, im_id)
+        im_data = _prepare_image_data_block(im_data, data_format)
+        im_data_block, data_size, codec_str = _serialize_data_block(
+            im_data, im_attrs, codec, level, shuffle
+        )
+
+        # Assemble location attribute, *provisional* until we can compute the data block position
+        im_attrs["location"] = XISF._to_location(("attachment", "", data_size))
+
+        # __/ Build (provisional) XML Header \__________
+        # (for attached data blocks, the location is provisional)
+        #   Convert metadata (dict) to XML Header
+        xisf_header_xml = ET.Element("xisf", XISF._xisf_attrs)
+
+        #   Image
+        image_xml = ET.SubElement(xisf_header_xml, "Image", im_attrs)
+
+        #     Image FITSKeywords
+        for kw_name, kw_values in image_metadata.get("FITSKeywords", {}).items():
+            XISF._insert_fitskeyword(image_xml, kw_name, kw_values)
+
+        # attached_blocks_locations will reference every element whose data block is to be attached
+        #   = [{"xml": ElementTree, "position": int, "size": int, "data": ndarray or str}]
+        #   (position key is actually a placeholder, it will be overwritten by
+        #   _compute_attached_positions)
+        # The first element is the image (*provisional* location):
+        attached_blocks_locations = [
+            {
+                "xml": image_xml,
+                "position": 0,
+                "size": data_size,
+                "data": im_data_block,
+            }
+        ]
+
+        #     Image XISFProperties
+        for p_dict in image_metadata.get("XISFProperties", {}).values():
+            if attached_block := XISF._insert_property(image_xml, p_dict, max_inline_blk_sz):
+                attached_blocks_locations.append(attached_block)
+
+        #   File Metadata
+        metadata_xml = ET.SubElement(xisf_header_xml, "Metadata")
+        _update_xisf_metadata(creator_app, blk_sz, max_inline_blk_sz, codec, level)
+        for property_dict in xisf_metadata.values():
+            if attached_block := XISF._insert_property(
+                metadata_xml, property_dict, max_inline_blk_sz
+            ):
+                attached_blocks_locations.append(attached_block)
+
+        # Header provisional size (without attachment positions)
+        xisf_header = ET.tostring(xisf_header_xml, encoding="utf8")
+        header_provisional_sz = (
+            len(XISF._signature) + XISF._headerlength_len + len(xisf_header) + XISF._reserved_len
+        )
+
+        # Update location for every block in attached_blocks_locations
+        _compute_attached_positions(header_provisional_sz, attached_blocks_locations)
+
+        with open(fname, "wb") as f:
+            # Write XISF signature
+            f.write(XISF._signature)
+
+            xisf_header = ET.tostring(xisf_header_xml, encoding="utf8")
+            headerlength = len(xisf_header)
+            # Write header length
+            f.write(headerlength.to_bytes(XISF._headerlength_len, byteorder="little"))
+
+            # Write reserved field
+            reserved_field = _zero_pad(XISF._reserved_len)
+            f.write(reserved_field)
+
+            # Write header
+            f.write(xisf_header)
+
+            # Write data blocks
+            for b in attached_blocks_locations:
+                pos, data_block = b["position"], b["data"]
+                f.write(_zero_pad(pos - f.tell()))
+                assert f.tell() == pos
+                f.write(data_block)
+            bytes_written = f.tell()
+
+        return bytes_written, codec_str
+
+    # __/ Auxiliary functions to handle XISF attributes \________
+
+    # Process property attributes and convert to dict
+    def _process_property(self, p_et):
+        p_dict = p_et.attrib.copy()
+
+        if p_dict["type"] == "TimePoint":
+            # Timepoint 'value' attribute already set (as str)
+            # Convert ISO 8601 string to datetime object
+            try:
+                tp_str = p_dict.get("value", "")
+                if tp_str:
+                    # Handle XISF TimePoint format: ISO 8601 with optional timezone
+                    # Examples: "2023-01-15T10:30:00Z", "2023-01-15T10:30:00.123456"
+                    tp_str = tp_str.replace("Z", "+00:00")
+                    if "." in tp_str and "+" not in tp_str.split(".")[-1] and "-" not in tp_str.split(".")[-1]:
+                        # Add UTC timezone if missing after fractional seconds
+                        tp_str += "+00:00"
+                    p_dict["datetime"] = datetime.fromisoformat(tp_str)
+            except (ValueError, TypeError):
+                # Keep original string value if parsing fails
+                p_dict["datetime"] = None
+        elif p_dict["type"] == "String":
+            # NOTE: currently does: p_dict["value"] = p_et.text; then if location -> read block now
+            p_dict["value"] = p_et.text  # may be None
+            if "location" in p_dict:
+                self._process_location_compression(p_dict)
+                # LAZY: do NOT read block here
+                return _make_lazy(p_dict)
+            return p_dict
+        elif p_dict["type"] == "Boolean":
+            # Boolean valid values are "true" and "false"
+            p_dict["value"] = p_dict["value"] == "true"
+        elif "value" in p_et.attrib:
+            # Scalars (Float64, UInt32, etc.) and Complex*
+            p_dict["value"] = ast.literal_eval(p_dict["value"])
+        elif "Vector" in p_dict["type"]:
+            p_dict["value"] = p_et.text
+            p_dict["length"] = int(p_dict["length"])
+            p_dict["dtype"] = self._parse_vector_dtype(p_dict["type"])
+            self._process_location_compression(p_dict)
+            # LAZY: do NOT read block here
+            return _make_lazy(p_dict)
+
+        elif "Matrix" in p_dict["type"]:
+            p_dict["value"] = p_et.text
+            p_dict["rows"] = int(p_dict["rows"])
+            p_dict["columns"] = int(p_dict["columns"])
+            p_dict["dtype"] = self._parse_vector_dtype(p_dict["type"])
+            self._process_location_compression(p_dict)
+            # LAZY: do NOT read block here
+            return _make_lazy(p_dict)
+        else:
+            print(f"Unsupported Property type {p_dict['type']}: {p_et}")
+            p_dict = False
+
+        return p_dict
+
+    def resolve_property(self, p_dict):
+        """
+        Resolve a lazy property (String/Vector/Matrix with a data block).
+        Mutates p_dict in place and returns decoded 'value'.
+        """
+        if not p_dict.get("_lazy"):
+            return p_dict.get("value")
+
+        raw = self._read_data_block(p_dict)
+
+        t = p_dict["type"]
+        if t == "String":
+            val = raw.decode("utf-8")
+        elif "Vector" in t:
+            val = np.frombuffer(raw, dtype=p_dict["dtype"], count=p_dict["length"])
+        elif "Matrix" in t:
+            length = p_dict["rows"] * p_dict["columns"]
+            val = np.frombuffer(raw, dtype=p_dict["dtype"], count=length).reshape((p_dict["rows"], p_dict["columns"]))
+        else:
+            # if something else ever gets marked lazy
+            val = raw
+
+        p_dict["value"] = val
+        p_dict["_lazy"] = False
+        return val
+
+    def can_partial_read_image(self, n=0):
+        meta = self._images_meta[n]
+        if meta["location"][0] != "attachment":
+            return False
+        if "compression" in meta:
+            return False
+        return True
+
+    def read_image_roi(self, n=0, x0=0, y0=0, x1=None, y1=None, channels=None, data_format="channels_last"):
+        meta = self._images_meta[n]
+        if meta["location"][0] != "attachment":
+            raise NotImplementedError("ROI read only supported for attachment blocks")
+        if "compression" in meta:
+            raise NotImplementedError("ROI read not supported for compressed image blocks")
+
+        w, h, chc = meta["geometry"]
+        dtype = meta["dtype"]
+        itemsize = dtype.itemsize
+
+        if x1 is None: x1 = w
+        if y1 is None: y1 = h
+        x0 = max(0, min(w, x0)); x1 = max(0, min(w, x1))
+        y0 = max(0, min(h, y0)); y1 = max(0, min(h, y1))
+        if x1 <= x0 or y1 <= y0:
+            raise ValueError("Empty ROI")
+
+        if channels is None:
+            channels = list(range(chc))
+        else:
+            channels = list(channels)
+
+        _, pos, _size = meta["location"]
+        roi_w = x1 - x0
+        roi_h = y1 - y0
+
+        out = np.empty((len(channels), roi_h, roi_w), dtype=dtype)
+
+        row_bytes = w * itemsize
+        roi_bytes = roi_w * itemsize
+        plane_bytes = h * row_bytes
+
+        with open(self._fname, "rb") as f:
+            for ci, c in enumerate(channels):
+                if c < 0 or c >= chc:
+                    raise IndexError(f"channel {c} out of range")
+                plane_base = pos + c * plane_bytes
+                for r, y in enumerate(range(y0, y1)):
+                    offset = plane_base + y * row_bytes + x0 * itemsize
+                    f.seek(offset)
+                    out[ci, r, :] = np.frombuffer(f.read(roi_bytes), dtype=dtype, count=roi_w)
+
+        if data_format == "channels_last":
+            return np.transpose(out, (1, 2, 0))
+        return out
+
+
+    @staticmethod
+    def _process_location_compression(p_dict):
+        p_dict["location"] = XISF._parse_location(p_dict["location"])
+        if "compression" in p_dict:
+            p_dict["compression"] = XISF._parse_compression(p_dict["compression"])
+
+    # Insert XISF properties in the XML tree
+    @staticmethod
+    def _insert_property(parent, p_dict, max_inline_block_size, codec=None, shuffle=False):
+        """Insert a property into the XML tree.
+        
+        Args:
+            parent: Parent XML element
+            p_dict: Property dictionary with 'id', 'type', 'value', and optional 'format', 'comment'
+            max_inline_block_size: Maximum size for inline data blocks
+            codec: Compression codec (None, 'zlib', 'lz4', 'lz4hc', 'zstd')
+            shuffle: Enable byte shuffling for compression
+        """
+        scalars = ["Int", "Byte", "Short", "Float", "Boolean", "TimePoint"]
+        
+        # Build base attributes including optional format and comment
+        def _build_attrs(base_attrs):
+            attrs = dict(base_attrs)
+            if "format" in p_dict and p_dict["format"]:
+                attrs["format"] = str(p_dict["format"])
+            if "comment" in p_dict and p_dict["comment"]:
+                attrs["comment"] = str(p_dict["comment"])
+            return attrs
+
+        if any(t in p_dict["type"] for t in scalars):
+            # scalars and TimePoint
+            value_str = str(p_dict["value"])
+            # Boolean requires lowercase per XISF spec
+            if p_dict["type"] == "Boolean":
+                value_str = "true" if p_dict["value"] else "false"
+            attrs = _build_attrs({
+                "id": p_dict["id"],
+                "type": p_dict["type"],
+                "value": value_str,
+            })
+            ET.SubElement(parent, "Property", attrs)
+        elif p_dict["type"] == "String":
+            text = str(p_dict["value"])
+            data_bytes = text.encode("utf-8")
+            sz = len(data_bytes)
+            if sz > max_inline_block_size:
+                # Attach string as data block with optional compression
+                attrs = _build_attrs({
+                    "id": p_dict["id"],
+                    "type": p_dict["type"],
+                })
+                if codec:
+                    compressed, comp_str = XISF._compress_data_block(data_bytes, codec, shuffle, 1)
+                    attrs["location"] = XISF._to_location(("attachment", "", len(compressed)))
+                    attrs["compression"] = comp_str
+                    xml = ET.SubElement(parent, "Property", attrs)
+                    return {"xml": xml, "location": 0, "size": len(compressed), "data": compressed}
+                else:
+                    attrs["location"] = XISF._to_location(("attachment", "", sz))
+                    xml = ET.SubElement(parent, "Property", attrs)
+                    return {"xml": xml, "location": 0, "size": sz, "data": data_bytes}
+            else:
+                # string directly as child (no 'location' attribute)
+                attrs = _build_attrs({
+                    "id": p_dict["id"],
+                    "type": p_dict["type"],
+                })
+                ET.SubElement(parent, "Property", attrs).text = text
+        elif "Vector" in p_dict["type"]:
+            data = p_dict["value"]
+            raw_bytes = data.tobytes()
+            sz = len(raw_bytes)
+            item_size = data.itemsize
+            if sz > max_inline_block_size:
+                # Attach vector as data block with optional compression
+                attrs = _build_attrs({
+                    "id": p_dict["id"],
+                    "type": p_dict["type"],
+                    "length": str(data.size),
+                })
+                if codec:
+                    compressed, comp_str = XISF._compress_data_block(raw_bytes, codec, shuffle, item_size)
+                    attrs["location"] = XISF._to_location(("attachment", "", len(compressed)))
+                    attrs["compression"] = comp_str
+                    xml = ET.SubElement(parent, "Property", attrs)
+                    return {"xml": xml, "location": 0, "size": len(compressed), "data": compressed}
+                else:
+                    attrs["location"] = XISF._to_location(("attachment", "", sz))
+                    xml = ET.SubElement(parent, "Property", attrs)
+                    return {"xml": xml, "location": 0, "size": sz, "data": data}
+            else:
+                # Inline data block (assuming base64)
+                attrs = _build_attrs({
+                    "id": p_dict["id"],
+                    "type": p_dict["type"],
+                    "length": str(data.size),
+                    "location": XISF._to_location(("inline", "base64")),
+                })
+                ET.SubElement(parent, "Property", attrs).text = str(base64.b64encode(data.tobytes()), "ascii")
+        elif "Matrix" in p_dict["type"]:
+            data = p_dict["value"]
+            raw_bytes = data.tobytes()
+            sz = len(raw_bytes)
+            item_size = data.itemsize
+            if sz > max_inline_block_size:
+                # Attach matrix as data block with optional compression
+                attrs = _build_attrs({
+                    "id": p_dict["id"],
+                    "type": p_dict["type"],
+                    "rows": str(data.shape[0]),
+                    "columns": str(data.shape[1]),
+                })
+                if codec:
+                    compressed, comp_str = XISF._compress_data_block(raw_bytes, codec, shuffle, item_size)
+                    attrs["location"] = XISF._to_location(("attachment", "", len(compressed)))
+                    attrs["compression"] = comp_str
+                    xml = ET.SubElement(parent, "Property", attrs)
+                    return {"xml": xml, "location": 0, "size": len(compressed), "data": compressed}
+                else:
+                    attrs["location"] = XISF._to_location(("attachment", "", sz))
+                    xml = ET.SubElement(parent, "Property", attrs)
+                    return {"xml": xml, "location": 0, "size": sz, "data": data}
+            else:
+                # Inline data block (assuming base64)
+                attrs = _build_attrs({
+                    "id": p_dict["id"],
+                    "type": p_dict["type"],
+                    "rows": str(data.shape[0]),
+                    "columns": str(data.shape[1]),
+                    "location": XISF._to_location(("inline", "base64")),
+                })
+                ET.SubElement(parent, "Property", attrs).text = str(base64.b64encode(data.tobytes()), "ascii")
+        else:
+            print(f"Warning: skipping unsupported property {p_dict}")
+
+        return False
+
+    # Insert FITS Keywords in the XML tree
+    @staticmethod
+    def _insert_fitskeyword(image_xml, keyword_name, keyword_values):
+        for entry in keyword_values:
+            ET.SubElement(
+                image_xml,
+                "FITSKeyword",
+                {
+                    "name": keyword_name,
+                    "value": entry["value"],
+                    "comment": entry["comment"],
+                },
+            )
+
+    # Returns image shape, e.g. (x, y, channels)
+    @staticmethod
+    def _parse_geometry(g):
+        return tuple(map(int, g.split(":")))
+
+    # Returns ("attachment", position, size), ("inline", encoding) or ("embedded")
+    @staticmethod
+    def _parse_location(l):
+        ll = l.split(":")
+        if ll[0] not in ["inline", "embedded", "attachment"]:
+            raise NotImplementedError(f"Data block location type '{ll[0]}' not implemented")
+        return (ll[0], int(ll[1]), int(ll[2])) if ll[0] == "attachment" else ll
+
+    # Serialize location tuple to string, as value for location attribute
+    @staticmethod
+    def _to_location(location_tuple):
+        return ":".join([str(e) for e in location_tuple])
+
+    # Returns (codec, uncompressed_size, item_size); item_size is None if not using byte shuffling
+    @staticmethod
+    def _parse_compression(c):
+        cl = c.split(":")
+        if len(cl) == 3:
+            # (codec+byteshuffling, uncompressed_size, shuffling_item_size)
+            return (cl[0], int(cl[1]), int(cl[2]))
+        else:
+            # (codec, uncompressed_size, None)
+            return (cl[0], int(cl[1]), None)
+
+    # Return equivalent numpy dtype
+    @staticmethod
+    def _parse_sampleFormat(s):
+        # Translate alternate names to "canonical" type names
+        alternate_names = {
+            'Byte': 'UInt8',
+            'Short': 'Int16',
+            'UShort': 'UInt16',
+            'Int': 'Int32',
+            'UInt': 'UInt32',
+            'Float': 'Float32',
+            'Double': 'Float64',
+        }
+        try:
+            s = alternate_names[s]
+        except KeyError:
+            pass
+
+        _dtypes = {
+            "UInt8": np.dtype("uint8"),
+            "UInt16": np.dtype("uint16"),
+            "UInt32": np.dtype("uint32"),
+            "Float32": np.dtype("float32"),
+            "Float64": np.dtype("float64"),
+        }
+        try:
+            return _dtypes[s]
+        except:
+            raise NotImplementedError(f"sampleFormat {s} not implemented")
+
+    # Return XISF data type from numpy dtype
+    @staticmethod
+    def _get_sampleFormat(dtype):
+        _sampleFormats = {
+            "uint8": "UInt8",
+            "uint16": "UInt16",
+            "uint32": "UInt32",
+            "float32": "Float32",
+            "float64": "Float64",
+        }
+        try:
+            return _sampleFormats[str(dtype)]
+        except:
+            raise NotImplementedError(f"sampleFormat for {dtype} not implemented")
+
+    @staticmethod
+    def _parse_vector_dtype(type_name):
+        # Translate alternate names to "canonical" type names
+        alternate_names = {
+            'ByteArray': 'UI8Vector',
+            'IVector': 'I32Vector',
+            'UIVector': 'UI32Vector',
+            'Vector': 'F64Vector',
+        }
+        try:
+            type_name = alternate_names[type_name]
+        except KeyError:
+            pass
+
+        type_prefix = type_name[:-6]  # removes "Vector" and "Matrix" suffixes
+        _dtypes = {
+            "I8": np.dtype("int8"),
+            "UI8": np.dtype("uint8"),
+            "I16": np.dtype("int16"),
+            "UI16": np.dtype("uint16"),
+            "I32": np.dtype("int32"),
+            "UI32": np.dtype("uint32"),
+            "I64": np.dtype("int64"),
+            "UI64": np.dtype("uint64"),
+            "F32": np.dtype("float32"),
+            "F64": np.dtype("float64"),
+            "C32": np.dtype("csingle"),
+            "C64": np.dtype("cdouble"),
+        }
+        try:
+            return _dtypes[type_prefix]
+        except:
+            raise NotImplementedError(f"data type {type_name} not implemented")
+
+    # __/ Auxiliary functions for compression/shuffling \________
+
+    # Un-byteshuffling implementation based on numpy
+    @staticmethod
+    def _unshuffle(d, item_size):
+        a = np.frombuffer(d, dtype=np.dtype("uint8"))
+        a = a.reshape((item_size, -1))
+        return np.transpose(a).tobytes()
+
+    # Byteshuffling implementation based on numpy
+    @staticmethod
+    def _shuffle(d, item_size):
+        a = np.frombuffer(d, dtype=np.dtype("uint8"))
+        a = a.reshape((-1, item_size))
+        return np.transpose(a).tobytes()
+
+    # LZ4/zlib/zstd decompression
+    @staticmethod
+    def _decompress(data, elem):
+        # (codec, uncompressed-size, item-size); item-size is None if not using byte shuffling
+        codec, uncompressed_size, item_size = elem["compression"]
+
+        if codec.startswith("lz4"):
+            data = lz4.block.decompress(data, uncompressed_size=uncompressed_size)
+        elif codec.startswith("zstd"):
+            data = zstandard.decompress(data, max_output_size=uncompressed_size)
+        elif codec.startswith("zlib"):
+            data = zlib.decompress(data)
+        else:
+            raise NotImplementedError(f"Unimplemented compression codec {codec}")
+
+        if item_size:  # using byte-shuffling
+            data = XISF._unshuffle(data, item_size)
+
+        return data
+
+    @staticmethod
+    def _compress_data_block(data, codec, shuffle=False, itemsize=1):
+        """Compress a data block and return (compressed_bytes, compression_attr_string).
+        
+        Args:
+            data: bytes or numpy array to compress
+            codec: 'zlib', 'lz4', 'lz4hc', or 'zstd'
+            shuffle: enable byte shuffling
+            itemsize: item size for byte shuffling (1 for strings, dtype.itemsize for arrays)
+        
+        Returns:
+            tuple: (compressed_bytes, compression_attribute_string)
+        """
+        if hasattr(data, 'tobytes'):
+            raw_bytes = data.tobytes()
+        else:
+            raw_bytes = bytes(data)
+        
+        uncompressed_size = len(raw_bytes)
+        compressed = XISF._compress(raw_bytes, codec, shuffle=shuffle, itemsize=itemsize if shuffle else None)
+        
+        # Build compression attribute string: "codec:uncompressed_size" or "codec+sh:uncompressed_size:itemsize"
+        if shuffle and itemsize > 1:
+            comp_str = f"{codec}+sh:{uncompressed_size}:{itemsize}"
+        else:
+            comp_str = f"{codec}:{uncompressed_size}"
+        
+        return compressed, comp_str
+
+    # LZ4/zlib/zstd compression
+    @staticmethod
+    def _compress(data, codec, level=None, shuffle=False, itemsize=None):
+        compressed = XISF._shuffle(data, itemsize) if shuffle else data
+
+        if codec == "lz4hc":
+            level = level if level else XISF._compression_def_level["lz4hc"]
+            compressed = lz4.block.compress(
+                compressed, mode="high_compression", compression=level, store_size=False
+            )
+        elif codec == "lz4":
+            compressed = lz4.block.compress(compressed, store_size=False)
+        elif codec == "zstd":
+            level = level if level else XISF._compression_def_level["zstd"]
+            compressed = zstandard.compress(compressed, level=level)
+        elif codec == "zlib":
+            level = level if level else XISF._compression_def_level["zlib"]
+            compressed = zlib.compress(compressed, level=level)
+        else:
+            raise NotImplementedError(f"Unimplemented compression codec {codec}")
+
+        return compressed