volsegtools 0.0.0__py3-none-any.whl

volsegtools/preprocessor/preprocessor.py

@@ -0,0 +1,75 @@
+import asyncio
+from pathlib import Path
+from typing import List, Optional
+
+from volsegtools.abc import Converter, Downsampler
+from volsegtools.core.lattice_kind import LatticeKind
+from volsegtools.model.working_store import WorkingStore
+from volsegtools.serialization import BCIFSerializer
+
+
+class Preprocessor:
+    state = {"downsampling_status": 0.0}
+
+    def __init__(
+        self,
+        downsampler: Downsampler,
+        converter: Converter,
+        volume_input_files: List[Path],
+        segmentation_input_files: List[Path],
+        metadata_input_files: Optional[List[Path]] = None,
+        annotations_input_files: Optional[List[Path]] = None,
+        work_dir: Optional[Path] = None,
+        output_dir: Optional[Path] = None,
+    ):
+        self.converter: Converter = converter
+        self._output_dir = output_dir if output_dir is not None else Path()
+        self._data = WorkingStore.instance
+        self.downsampler = downsampler
+        self._volume_input_files = volume_input_files
+        self._segmentation_input_files = segmentation_input_files
+        self._metadata_input_files = metadata_input_files
+        self._annotations_input_files = annotations_input_files
+        self._work_dir = work_dir
+
+    async def transform_volume(self):
+        raise NotImplementedError()
+
+    async def collect_metadata(self):
+        raise NotImplementedError()
+
+    def create_converter_from_format(self):
+        raise NotImplementedError()
+
+    async def transform_segmentation(self):
+        raise NotImplementedError()
+
+    async def preprocess(self):
+        raise NotImplementedError()
+
+    def sync_preprocess(self):
+        data = []
+
+        # 1. Data conversion phase
+        for path in self._volume_input_files:
+            data.append(asyncio.run(self.converter.transform_volume(path)))
+            data[-1].metadata = asyncio.run(self.converter.collect_metadata(path))
+            # FIX: This should be automatic!
+            data[-1].metadata.kind = LatticeKind.VOLUME
+        for path in self._segmentation_input_files:
+            data.append(asyncio.run(self.converter.transform_segmentation(path)))
+            data[-1].metadata = asyncio.run(self.converter.collect_metadata(path))
+            # FIX: This should be automatic!
+            data[-1].metadata.kind = LatticeKind.SEGMENTATION
+
+        downsampled_data = []
+        for ref in data:
+            # TODO: it has to be flattened!
+            downsampled_data += asyncio.run(self.downsampler.downsample_lattice(ref))
+        print("Downsampled Data:", downsampled_data)
+        # 4. Serialization Phase
+        for ref in downsampled_data:
+            asyncio.run(BCIFSerializer.serialize(ref, self._output_dir))
+
+    def downsample(self):
+        pass

volsegtools/preprocessor/preprocessor_builder.py

@@ -0,0 +1,110 @@
+from pathlib import Path
+from typing import List
+
+from typing_extensions import Self
+
+from volsegtools.abc import Converter, Downsampler
+from volsegtools.preprocessor import Preprocessor
+
+
+class PreprocessorBuilder:
+    """Allows fine-grained specification of the preprocessor."""
+
+    def __init__(self) -> None:
+        self._work_dir: Path | None = None
+        self._output_dir: Path | None = None
+        self._volume_sources: List[Path] = []
+        self._segmentation_sources: List[Path] = []
+        self._annotations_sources: List[Path] = []
+        self._metadata_sources: List[Path] = []
+        self._converter: Converter | None = None
+        self._downsampler: Downsampler | None = None
+        # Currently, we only support a single preprocessor, after that we
+        # could add support for generic preprocessor
+        # self._preprocessor_type: Optional[Preprocessor] = None
+
+    def set_converter(self, converter: Converter) -> Self:
+        """Sets a converter that is going to be used by the resulting
+        preprocessor.
+
+        If this method is going to be called multimple times, it is going
+        to override the previously set converter.
+        """
+        self._converter = converter
+        return self
+
+    def set_downsampler(self, downsampler: Downsampler) -> Self:
+        """Sets a downsampler that is going to be used by the resulting
+        preprocessor.
+
+        If this method is going to be called multimple times, it is going
+        to override the previously set downsampler.
+        """
+        self._downsampler = downsampler
+        return self
+
+    def add_volume_src_file(self, file_path: Path) -> Self:
+        """Adds the source file for volumetric data.
+
+        It should be used in cases where there are multiple files and
+        set_input_file cannot be used.
+        """
+        self._volume_sources.append(file_path)
+        return self
+
+    def add_segmentation_src_file(self, file_path: Path) -> Self:
+        """Sets the source file for annotations.
+
+        It should be used in cases where there are multiple files and
+        set_input_file cannot be used.
+        """
+        self._segmentation_sources.append(file_path)
+        return self
+
+    def add_metadata_src_file(self, file_path: Path) -> Self:
+        """Adds the source file for metadata.
+
+        It should be used in cases where there are multiple files and
+        set_input_file cannot be used.
+        """
+        return self
+
+    def add_annotations_src_file(self, file_path: Path) -> Self:
+        """Adds the source file for annotations.
+
+        It should be used in cases where there are multiple files and
+        set_input_file cannot be used.
+        """
+        return self
+
+    def set_work_dir(self, file_path: Path) -> Self:
+        """Sets the working directory of the processor.
+
+        It is going to be added to the search path for source files of the
+        preprocessor. Also, the output of the preprocessor is going to be
+        saved at this location.
+        """
+        return self
+
+    def set_output_dir(self, file_path: Path) -> Self:
+        self._output_dir = file_path
+        return self
+
+    def build(self) -> Preprocessor:
+        """Builds the resulting preprocessor."""
+        if self._downsampler is None:
+            raise RuntimeError("Downsampler was not set")
+
+        if self._converter is None:
+            raise RuntimeError("Converter was not set")
+
+        return Preprocessor(
+            self._downsampler,
+            self._converter,
+            self._volume_sources,
+            self._segmentation_sources,
+            self._metadata_sources,
+            self._annotations_sources,
+            self._work_dir,
+            self._output_dir,
+        )

volsegtools/serialization/__init__.py

@@ -0,0 +1 @@
+from .bcif_serializer import BCIFSerializer

volsegtools/serialization/bcif_serializer.py

@@ -0,0 +1,318 @@
+import dataclasses
+import sys
+from pathlib import Path
+from typing import Collection
+
+import ciftools
+import ciftools.serialization
+import numpy as np
+from ciftools.binary.decoder import ByteArrayEncoding, DataType
+from ciftools.binary.encoder import BYTE_ARRAY, BinaryCIFEncoder, DataTypeEnum
+from ciftools.binary.writer import EncodedCIFData
+from ciftools.models.writer import CIFCategoryDesc as CategoryDesc
+from ciftools.models.writer import CIFFieldDesc as Field
+
+from volsegtools.abc import Serializer
+from volsegtools.model import (
+    ChannelMetadata,
+    OpaqueDataHandle,
+    OriginalTimeFrameMetadata,
+    TimeFrameMetadata,
+)
+from volsegtools.model.working_store import WorkingStore
+
+
+@dataclasses.dataclass
+class VolumeDataBatch:
+    original_metadata: OriginalTimeFrameMetadata
+    target_metadata: TimeFrameMetadata
+    channel: ChannelMetadata
+
+
+class DummyVolumeServerEncoder(BinaryCIFEncoder):
+    def encode(self, data: np.ndarray) -> EncodedCIFData:
+        data_type: DataTypeEnum = DataType.from_dtype(data.dtype)
+        encoding: ByteArrayEncoding = {
+            "kind": "VolumeServer",
+            "type": data_type,
+        }
+
+        bo = data.dtype.byteorder
+        if bo == ">" or (bo == "=" and sys.byteorder == "big"):
+            new_bo = data.dtype.newbyteorder("<")
+            data = np.array(data, dtype=new_bo)
+
+        return EncodedCIFData(data=data.tobytes(), encoding=[encoding])
+
+
+DUMMY_VOLUME_SERVER = DummyVolumeServerEncoder()
+
+
+class VolumeData3DInfoDesc(CategoryDesc):
+    name = "volume_data_3d_info"
+
+    @staticmethod
+    def get_row_count(_) -> int:
+        return 1
+
+    @staticmethod
+    def get_field_descriptors(data: VolumeDataBatch) -> Collection[Field]:
+        volume_server_encoder = lambda _: BYTE_ARRAY
+        return [
+            Field.strings(
+                name="name",
+                value=lambda d, i: str(data.target_metadata.id),
+            ),
+            Field.numbers(
+                name="axis_order[0]",
+                value=lambda d, i: data.target_metadata.axis_order.x,
+                encoder=volume_server_encoder,
+                dtype="i4",
+            ),
+            Field.numbers(
+                name="axis_order[1]",
+                value=lambda d, i: data.target_metadata.axis_order.y,
+                encoder=volume_server_encoder,
+                dtype="i4",
+            ),
+            Field.numbers(
+                name="axis_order[2]",
+                value=lambda d, i: data.target_metadata.axis_order.z,
+                encoder=volume_server_encoder,
+                dtype="i4",
+            ),
+            Field.numbers(
+                name="origin[0]",
+                value=lambda d, i: data.target_metadata.origin.x,
+                encoder=volume_server_encoder,
+                dtype="i4",
+            ),
+            Field.numbers(
+                name="origin[1]",
+                value=lambda d, i: data.target_metadata.origin.y,
+                encoder=volume_server_encoder,
+                dtype="i4",
+            ),
+            Field.numbers(
+                name="origin[2]",
+                value=lambda d, i: data.target_metadata.origin.z,
+                encoder=volume_server_encoder,
+                dtype="i4",
+            ),
+            Field.numbers(
+                name="dimensions[0]",
+                value=lambda d, i: 1,
+                encoder=volume_server_encoder,
+                dtype="i4",
+            ),
+            Field.numbers(
+                name="dimensions[1]",
+                value=lambda d, i: 1,
+                encoder=volume_server_encoder,
+                dtype="i4",
+            ),
+            Field.numbers(
+                name="dimensions[2]",
+                value=lambda d, i: 1,
+                encoder=volume_server_encoder,
+                dtype="i4",
+            ),
+            Field.numbers(
+                name="sample_rate",
+                value=lambda d, i: data.target_metadata.resolution,
+                encoder=volume_server_encoder,
+                dtype="i4",
+            ),
+            Field.numbers(
+                name="sample_count[0]",
+                value=lambda d, i: data.target_metadata.lattice_dimensions.x,
+                encoder=volume_server_encoder,
+                dtype="i4",
+            ),
+            Field.numbers(
+                name="sample_count[1]",
+                value=lambda d, i: data.target_metadata.lattice_dimensions.y,
+                encoder=volume_server_encoder,
+                dtype="i4",
+            ),
+            Field.numbers(
+                name="sample_count[2]",
+                value=lambda d, i: data.target_metadata.lattice_dimensions.z,
+                encoder=volume_server_encoder,
+                dtype="i4",
+            ),
+            Field.numbers(
+                name="spacegroup_number",
+                value=lambda d, i: 1,
+                encoder=volume_server_encoder,
+                dtype="i4",
+            ),
+            Field.numbers(
+                name="spacegroup_cell_size[0]",
+                value=lambda d, i: data.target_metadata.voxel_size.x,
+                encoder=volume_server_encoder,
+                dtype="f8",
+            ),
+            Field.numbers(
+                name="spacegroup_cell_size[1]",
+                value=lambda d, i: data.target_metadata.voxel_size.y,
+                encoder=volume_server_encoder,
+                dtype="f8",
+            ),
+            Field.numbers(
+                name="spacegroup_cell_size[2]",
+                value=lambda d, i: data.target_metadata.voxel_size.z,
+                encoder=volume_server_encoder,
+                dtype="f8",
+            ),
+            Field.numbers(
+                name="spacegroup_cell_angles[0]",
+                value=lambda d, i: 90,
+                encoder=volume_server_encoder,
+                dtype="f8",
+            ),
+            Field.numbers(
+                name="spacegroup_cell_angles[1]",
+                value=lambda d, i: 90,
+                encoder=volume_server_encoder,
+                dtype="f8",
+            ),
+            Field.numbers(
+                name="spacegroup_cell_angles[2]",
+                value=lambda d, i: 90,
+                encoder=volume_server_encoder,
+                dtype="f8",
+            ),
+            Field.numbers(
+                name="mean_source",
+                value=lambda d, i: data.channel.statistics.mean,
+                encoder=volume_server_encoder,
+                dtype="f8",
+            ),
+            Field.numbers(
+                name="mean_sampled",
+                value=lambda d, i: data.channel.statistics.mean,
+                encoder=volume_server_encoder,
+                dtype="f8",
+            ),
+            Field.numbers(
+                name="sigma_source",
+                value=lambda d, i: data.channel.statistics.std,
+                encoder=volume_server_encoder,
+                dtype="f8",
+            ),
+            Field.numbers(
+                name="sigma_sampled",
+                value=lambda d, i: data.channel.statistics.std,
+                encoder=volume_server_encoder,
+                dtype="f8",
+            ),
+            Field.numbers(
+                name="min_source",
+                value=lambda d, i: data.channel.statistics.min,
+                encoder=volume_server_encoder,
+                dtype="f8",
+            ),
+            Field.numbers(
+                name="min_sampled",
+                value=lambda d, i: data.channel.statistics.min,
+                encoder=volume_server_encoder,
+                dtype="f8",
+            ),
+            Field.numbers(
+                name="max_source",
+                value=lambda d, i: data.channel.statistics.max,
+                encoder=volume_server_encoder,
+                dtype="f8",
+            ),
+            Field.numbers(
+                name="max_sampled",
+                value=lambda d, i: data.channel.statistics.max,
+                encoder=volume_server_encoder,
+                dtype="f8",
+            ),
+        ]
+
+
+class DensityServerResultDesc(CategoryDesc):
+    name = "density_server_result"
+
+    @staticmethod
+    def get_row_count(_) -> int:
+        return 1
+
+    @staticmethod
+    def get_field_descriptors(data) -> Collection[Field]:
+        return [
+            Field.strings(name="query_type", value=lambda d, i: "box"),
+        ]
+
+
+class VolumeData3DDesc(CategoryDesc):
+    name = "volume_data_3d"
+
+    @staticmethod
+    def get_row_count(data: np.ndarray) -> int:
+        return data.size
+
+    @staticmethod
+    def get_field_descriptors(data: np.ndarray) -> Collection[Field]:
+        volume_server_encoder = lambda _: BYTE_ARRAY
+        return [
+            Field.number_array(
+                name="values",
+                array=lambda volume: volume,
+                encoder=volume_server_encoder,
+                dtype="f8",
+            ),
+        ]
+
+
+class BCIFSerializer(Serializer):
+    @staticmethod
+    async def serialize(data: OpaqueDataHandle, output_path: Path) -> None:
+        # This is currently working only for volumes!
+        for channel in data.metadata.channels:
+            data_batch = VolumeDataBatch(
+                # TODO: THERE HAS TO BE SOME EQUIVALENT
+                # data.metadata.original_time_frame,
+                data.metadata,
+                data.metadata,
+                channel,
+            )
+            writer = ciftools.serialization.create_binary_writer()
+
+            # We have to create the SERVER category, because it is required, we
+            # just have to say that it is a box
+            writer.start_data_block("SERVER")
+            # Adding a dummy
+            writer.write_category(DensityServerResultDesc, [np.arange(0)])
+
+            writer.start_data_block("VOLUME")
+            writer.write_category(VolumeData3DInfoDesc, [data_batch])
+
+            # TODO: prepend with WorkingStore path
+            metadata_file_name = Path(
+                "{}_r{}_tf{}_metadata.json".format(
+                    data.metadata.lattice_id,
+                    data.metadata.resolution,
+                    data.metadata.id,
+                )
+            )
+            metadata_file_name.write_text(str(dataclasses.asdict(data_batch)))
+
+            lattice = WorkingStore.instance.get_data_array(
+                data.metadata.lattice_id,
+                data.metadata.resolution,
+                data.metadata.id,
+                int(channel.id),
+            )
+            # We have to make the array 1D
+
+            print("COUNT BEFORE STORE:", np.count_nonzero(np.ravel(lattice)))
+            np.savetxt("data.csv", np.ravel(lattice), delimiter=",")
+
+            writer.write_category(VolumeData3DDesc, [np.ravel(lattice, "F")])
+
+            file_name = f"{data.metadata.lattice_id}_r{data.metadata.resolution}_tf{data.metadata.id}.bcif"
+            (output_path / file_name).write_bytes(writer.encode())

volsegtools/typing.py

@@ -0,0 +1,12 @@
+from typing import Union
+
+import numpy as np
+
+StorableDType = Union[
+    np.uint8,
+    np.uint16,
+    np.uint32,
+    np.int8,
+    np.int16,
+    np.int32,
+]

volsegtools-0.0.0.dist-info/METADATA

@@ -0,0 +1,22 @@
+Metadata-Version: 2.4
+Name: volsegtools
+Version: 0.0.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: dask
+Requires-Dist: dask_image
+Requires-Dist: zarr
+Requires-Dist: numpy
+Requires-Dist: mrcfile
+Requires-Dist: nibabel
+Requires-Dist: pydantic
+Requires-Dist: typer
+Requires-Dist: ciftools
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
+
+# Volseg Tools
+
+A library used for preprocessing of volumes and segmentations for further usage.

volsegtools-0.0.0.dist-info/RECORD

@@ -0,0 +1,38 @@
+volsegtools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+volsegtools/typing.py,sha256=-m0FLalCKGUT--tGct_pSRtw7JnUzaIBMh0BfcRVekM,156
+volsegtools/_cli/__init__.py,sha256=wIdwA2jyn2aw9LTygSdI35VEAwUpnNMQtM-aHei8o_8,38
+volsegtools/_cli/molstar_preprocessor.py,sha256=u_f9IlWwDpvJBojFowU1IrpxAjBkbGYCZWdZM5q69uI,2461
+volsegtools/abc/__init__.py,sha256=xJNQ2c2Fbw6G34CvD-hPhKluUB4Mb6aA5U1_oBIRVB4,218
+volsegtools/abc/converter.py,sha256=J-GocTeHAENHPfDq9CKl-voAcPcv8lg_kO6wOkFPs_0,1819
+volsegtools/abc/data_handle.py,sha256=0gBw0U6NdH6s0fcq3CLPG2iOEndWmJNYL6r3yrRNIwo,542
+volsegtools/abc/downsampler.py,sha256=uWb5nXBOldbIa0O023setj_IijIb4uTMBcLUn6c3GMQ,609
+volsegtools/abc/kernel.py,sha256=EQupdor1unWJL7E5Wyso_JVrKrArY-9DXZlFRcpmW78,135
+volsegtools/abc/preprocessor.py,sha256=_CJmI1ZA6Xqek5iKfLeEWE5f9Hserkhx2vGqNWVykZU,684
+volsegtools/abc/serializer.py,sha256=7rk0liOYn-wDhLKD_KYdqCnjK0pJZjWxTXXrVKnXjbI,307
+volsegtools/converter/__init__.py,sha256=1IAdRYtZ8E9SbnJ7xjYsuE7Deouaq3QfpHR6gcVaC6M,40
+volsegtools/converter/map_converter.py,sha256=aWEEtaPw6qxbnfA227FzwWyLxp3pD2-6c9D1MphKnd8,4822
+volsegtools/core/__init__.py,sha256=fww5lVXbV9I-zin5DbKv2izYXOdL5Fcn7HINVxqK1Zc,212
+volsegtools/core/bounds.py,sha256=WDkgVJhy_q40RMJtohkkMS2doro6jEjGamUQN8VGjQg,334
+volsegtools/core/downsampling_parameters.py,sha256=tv_A6nHHMWp6DcOaLkXLKrLqjAaZ66Yd1oL585fqWtA,821
+volsegtools/core/gaussian_kernel_3D.py,sha256=vUg3dosqm05_fY3C4j-YH-T0dTxG1hDy5MTXF_0JBIg,519
+volsegtools/core/lattice_kind.py,sha256=FqF-MCulLMOPajbmjKLTN8Vmj6WXPO8REE3miWZmXJk,93
+volsegtools/core/vector.py,sha256=U7F6WxtWHlTapsWG-v1JPKqIDtLwPoV3QZ1Z6rZBMuQ,138
+volsegtools/downsampler/__init__.py,sha256=f7V9d8-sPUOYAOf_ayFvSseAR4QMoXNIl6aIWMX0Otk,102
+volsegtools/downsampler/base_downsampler.py,sha256=zYMgnJgJjR5C-mvM1QssmU7wlYLxXPrqqfFAGPM-exc,655
+volsegtools/downsampler/hierarchy_downsampler.py,sha256=LeNNv2-7gHMxaZxQTzhZObDVilDvJdgSh_AlY_Vw6GE,9118
+volsegtools/model/__init__.py,sha256=JsoHfio1E9UdiFSGBXfH-NzKAhrkGbLlohcV24m2o-8,428
+volsegtools/model/chunking_mode.py,sha256=V6Bho2uvGvtX16Ggj9xdIQp7nwXyoJmSNwFmVboF7mg,185
+volsegtools/model/metadata.py,sha256=XJFdJGensZXt9v65BeU4Tzy55kPVJlCgbWbxgkxyYmc,1475
+volsegtools/model/opaque_data_handle.py,sha256=Mo_OzT49tdDoRyll5_71R_HF8-n7KwqlB7ZgrvqcxtE,3164
+volsegtools/model/storing_parameters.py,sha256=nz5Qgi4UJGck1-59NpASxeWdepFra79HkQ-BGYb0-NA,1907
+volsegtools/model/working_store.py,sha256=Rq5GL1rT0hRwKQ5QUe1hsYckFA7BBmQMEcc5FleYNi0,4706
+volsegtools/preprocessor/__init__.py,sha256=xErdaranOjR9Ea4accLRynbzfxWzDh_OovWWsR3SdsQ,93
+volsegtools/preprocessor/preprocessor.py,sha256=p3BiW6zb9jkMg-2MEnr_VKhy-esiB452srOKDWkkQgU,2747
+volsegtools/preprocessor/preprocessor_builder.py,sha256=QvhXNMPkkn6ce5TQ42mbfOnsAqYDF0b24h4nZ_lpSy4,3739
+volsegtools/serialization/__init__.py,sha256=Rg_vB2cJ0dj29B8kKT-chgyNTNbUmUa4tT5Pc6WqEZg,44
+volsegtools/serialization/bcif_serializer.py,sha256=o25p-XpIdKHEFBQeFVr-3RpbYTiVQ_EWK97yi5QP794,10878
+volsegtools-0.0.0.dist-info/METADATA,sha256=U4wyqLU0YndiwiM20caeL6SMI7s0LiJsnL9kAZqngis,571
+volsegtools-0.0.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+volsegtools-0.0.0.dist-info/entry_points.txt,sha256=REupaYQqcapvrApUyRFL1xFo4Iq9zD3KymP2n6PzOxQ,62
+volsegtools-0.0.0.dist-info/top_level.txt,sha256=xy_7ax33CZyn-ATVZRUtg12EkNct768TJgkVr5EQRCU,12
+volsegtools-0.0.0.dist-info/RECORD,,

volsegtools-0.0.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.10.2)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

volsegtools-0.0.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+molstar-preprocessor = volsegtools._cli:app

volsegtools-0.0.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+volsegtools