fits2tmp 0.4.0__tar.gz → 1.0.2__tar.gz

fits2tmp-1.0.2/PKG-INFO

@@ -0,0 +1,49 @@
+Metadata-Version: 2.4
+Name: fits2tmp
+Version: 1.0.2
+Summary: A way of loading FITS files into temporary filesystems to make them available in memory.
+Author: Verity Allan
+Author-email: Verity Allan <vla22@cam.ac.uk>
+License-Expression: BSD-3-Clause
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Requires-Dist: dask[distributed]>=2025.1.0,<2026
+Requires-Dist: astropy>=6.1.4,<7
+Requires-Dist: h5py>=3.16.0
+Requires-Dist: numpy>=2.1.2,<3
+Requires-Dist: pytest>=8.3.3,<9
+Requires-Dist: zarr>=3.4.1
+Requires-Python: >=3.12, <4
+Project-URL: Repository, https://gitlab.developers.cam.ac.uk/phy/ra/vla22-phd/fits2tmp
+Description-Content-Type: text/markdown
+
+# fits2tmp 
+
+This project allows you to take a large FITS file and render it as a set of smaller FITS files stored in-memory in a temporary directory, with appropriately updated headers.
+The program returns a list of the location of these files, which can then be processed by a parallel processing framework such as dask.
+
+The program also allows you to turn a zarr file containing your FITS data into small FITS files, provided you have extracted the FITS file header to a text file.
+Utilities in the program can be used to write headers out to or read them from a text file.
+
+## TODO:
+
+- Add a utility to convert FITS files to zarr, including saving the header out to a text file.
+- Add hdf5 support using a similar strategy to FITS.
+- Add xradio support.
+
+## Installing fits2tmp
+
+The program is available from pypi so can be installed with pip: `pip install fits2tmp`.
+
+## Example Usage
+
+```
+import fits2tmp as f2t
+
+a = fakefits2tmpfiles("fits", "/path/to/input/data", "/path/to/temporary/filesystem", "path/to/header/", (100, 100, 100), chunks=(20,20,20))
+```
+

fits2tmp-1.0.2/README.md

@@ -0,0 +1,26 @@
+# fits2tmp 
+
+This project allows you to take a large FITS file and render it as a set of smaller FITS files stored in-memory in a temporary directory, with appropriately updated headers.
+The program returns a list of the location of these files, which can then be processed by a parallel processing framework such as dask.
+
+The program also allows you to turn a zarr file containing your FITS data into small FITS files, provided you have extracted the FITS file header to a text file.
+Utilities in the program can be used to write headers out to or read them from a text file.
+
+## TODO:
+
+- Add a utility to convert FITS files to zarr, including saving the header out to a text file.
+- Add hdf5 support using a similar strategy to FITS.
+- Add xradio support.
+
+## Installing fits2tmp
+
+The program is available from pypi so can be installed with pip: `pip install fits2tmp`.
+
+## Example Usage
+
+```
+import fits2tmp as f2t
+
+a = fakefits2tmpfiles("fits", "/path/to/input/data", "/path/to/temporary/filesystem", "path/to/header/", (100, 100, 100), chunks=(20,20,20))
+```
+

fits2tmp-1.0.2/pyproject.toml

@@ -0,0 +1,38 @@
+[project]
+name = "fits2tmp"
+version = "1.0.2"
+description = "A way of loading FITS files into temporary filesystems to make them available in memory."
+requires-python = ">=3.12,<4"
+readme = "README.md"
+license = "BSD-3-Clause"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+]
+dependencies = [
+    "dask[distributed]>=2025.1.0,<2026",
+    "astropy>=6.1.4,<7",
+    "h5py>=3.16.0",
+    "numpy>=2.1.2,<3",
+    "pytest>=8.3.3,<9",
+    "zarr>=3.4.1",
+]
+
+[[project.authors]]
+name = "Verity Allan"
+email = "vla22@cam.ac.uk"
+
+[project.urls]
+Repository = "https://gitlab.developers.cam.ac.uk/phy/ra/vla22-phd/fits2tmp"
+
+[build-system]
+requires = ["uv_build>=0.13.5,<0.14.0"]
+build-backend = "uv_build"
+
+[tool.pytest.ini_options]
+addopts = ["--import-mode=importlib"]
+pythonpath = "src/fits2tmp/"

fits2tmp-1.0.2/pyproject.toml.orig

@@ -0,0 +1,37 @@
+[project]
+name = "fits2tmp"
+version = "1.0.2"
+description = "A way of loading FITS files into temporary filesystems to make them available in memory."
+authors = [{ name = "Verity Allan", email = "vla22@cam.ac.uk" }]
+requires-python = ">=3.12,<4"
+readme = "README.md"
+license = "BSD-3-Clause"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+]
+dependencies = [
+    "dask[distributed]>=2025.1.0,<2026",
+    "astropy>=6.1.4,<7",
+    "h5py>=3.16.0",
+    "numpy>=2.1.2,<3",
+    "pytest>=8.3.3,<9",
+    "zarr>=3.4.1",
+]
+
+[project.urls]
+Repository = "https://gitlab.developers.cam.ac.uk/phy/ra/vla22-phd/fits2tmp"
+
+[build-system]
+requires = ["uv_build>=0.13.5,<0.14.0"]
+build-backend = "uv_build"
+
+[tool.pytest.ini_options]
+addopts = [
+    "--import-mode=importlib",
+]
+pythonpath = "src/fits2tmp/"

fits2tmp-1.0.2/src/fits2tmp/fits2tmp.py

@@ -0,0 +1,173 @@
+import h5py
+import zarr
+import json
+from astropy.io import fits
+from astropy import wcs
+import dask.array as da
+from dask import delayed
+from dask.distributed import Lock
+import numpy as np
+import os
+import dask 
+import tempfile
+
+def arr_size_check(source_arr, desired_arr: tuple) -> bool:
+    if len(source_arr.shape) < len(desired_arr):
+        raise ValueError("You've asked for an array with more dimensions than the array stored")
+    else:
+        for i in range(min(len(source_arr.shape), len(desired_arr))):
+            # using min here allows us to avoid an IndexError when we have more dimensions in the source array
+            j = source_arr.shape[i] - desired_arr[i]
+            if j < 0:
+                raise IndexError("Desired array size is larger than the stored array in at least position %s: source array has dimensions %s, and you have requested $s", i, source_arr.shape, desired_arr)
+        return True
+
+#def fits_add_header(orig_header, sliced_data) -> fits.Header:
+#    coords = wcs.WCS(orig_header)
+#    new_header = orig_header
+#    new_header.update(coords[data_sli].to_header())
+#    return new_header
+
+def header_dict(header) -> dict:
+    d = {}
+    for i in header.cards:
+        d[i.keyword] = i.value
+    return d
+
+def dict2header(input_dict: dict) -> fits.Header:
+    nh = fits.Header(cards=input_dict)
+    return nh
+
+def dict2file(input_dict: dict, filepath):
+    with open(filepath, 'w') as f:
+        f.write(json.dumps(input_dict))
+
+def file2dict(input_file, filepath) -> dict:
+    with open(filepath, 'r') as j:
+        contents = json.loads(j.read())
+    return contents
+#filepath = []
+
+def fun(darr, destination, header, block_info=None, filepath=[]):
+    arr_loc = block_info[None]['array-location']
+    print(arr_loc)
+    ch_size = block_info[None]['chunk-shape']
+    chunk_loc = block_info[None]['chunk-location']
+    coords = []
+    for i in range(len(arr_loc)):
+        coords.append(slice(arr_loc[i][0],arr_loc[i][1]))
+    block_id = str(chunk_loc).replace(',', '-').replace('(', '-').replace(')', '').replace(' ', '')
+    n_header = dict2header(header_dict(header))
+    wcs_obj = wcs.WCS(n_header)
+    n_header.update(wcs_obj[coords].to_header())
+    out = fits.PrimaryHDU(data=darr, header=n_header)
+    outpath = destination.name  + '/' + block_id + ".fits"
+    out.writeto(outpath)
+    lock = Lock()
+    with lock:
+        filepath.append(outpath)
+    return darr 
+
+def fits2tmpfiles(inputpath, destination, chunks):
+    f = fits.open(inputpath)
+    destination = tempfile.TemporaryDirectory(dir=destination, delete=False)
+    d = da.from_array(f[0].data, chunks=chunks)
+    # TODO overlap
+    de = da.overlap.overlap(d, depth=None, boundary=None)
+    da.map_blocks(fun, d, destination, f[0].header, dtype=float).compute()
+    print(filepath)
+
+    return filepath
+
+def disk2dah(filetype, inputpath, destination, headerloc, arraysize, chunks=None):
+    # TODO: ADD code to handle arraysize differing from size of array in file
+    # maybe extend arr_size_check()?
+    # TODO make this support arrays that have other than 3 dims
+    destination = tempfile.TemporaryDirectory(dir=destination, delete=False)
+    match filetype:
+        case "fits":
+            f = fits.open(inputpath)
+            d = da.from_array(f[0].data[0:arraysize[0], 0:arraysize[1], 0:arraysize[2]], chunks=chunks)
+            n_header = dict2header(header_dict(f[0].header))
+            wcs_obj = wcs.WCS(n_header)
+        case "zarr":
+            d = da.from_zarr(inputpath, chunks=chunks)[0:arraysize, 0:arraysize[1], 0:arraysize[2]]
+            n_header = dict2header(file2dict(inputpath, headerloc))
+            wcs_obj = wcs.WCS(n_header)
+        case "hdf5":
+            f = h5py.File(filepath, "r")
+            #note this is hardcoded for the particular hdf5 files tested - needs parameterisation
+            d = da.from_array(f["array"][arraysize], chunks=chunks)
+            n_header = dict2header(file2dict(inputpath, headerloc))
+            wcs_obj = wcs.WCS(n_header)
+
+        case "xradio":
+            weeble
+        case _:
+            raise ValueError("The specified file type is not supported by fits2tmp.")
+    return [d, destination, n_header, wcs_obj]
+
+def fakefits2tmpfiles(filetype, inputpath, destination, arraysize, headerloc=None, chunks=None, overlap=None, fpaths=[]):
+    inputarray = disk2dah(filetype, inputpath, destination, headerloc, arraysize, chunks)
+    if overlap == None:
+        overlap = [0,0,0]
+    darr = inputarray[0]
+    print(darr.shape)
+    print(darr)
+    destination = inputarray[1]
+    header = inputarray[2]
+    print(header)
+    wcs_obj = inputarray[3]
+    print(len(darr.chunks))
+    # why am I pushing this array to delayed? ans: to parallelise construction of the list
+    #dd = darr.to_delayed()
+    for i in range(len(darr.chunks[0])):
+        x = i * chunks[0] - overlap[0] 
+        x_delt = (i* chunks[0]) + chunks[0] + overlap[0] 
+        # range check - when asking for overlap, we curtail at the edges of the array
+        if x < 0:
+            x = 0
+        if x_delt > arraysize[0] -1:
+            x_delt = arraysize[0]
+        print(x, x_delt)
+        for j in range(len(darr.chunks[1])):
+            y = j * chunks[1] - overlap[1]
+            y_delt = (j * chunks[1]) + chunks[1] + overlap[1] 
+            if y < 0:
+                y = 0
+            if y_delt > arraysize[1] -1:
+                y_delt = arraysize[1]
+            if len(darr.shape) == 2:
+                dest = str(i) + str(j)
+                data = dd[i][j]
+                wcs_coords = wcs_obj.slice((slice(x,x_delt), slice(y,y_delt)))
+            elif len(darr.shape) > 2:
+                for k in range(len(darr.chunks[2])):
+                    z = k * chunks[2] - overlap[2]
+                    z_delt = (k * chunks[2]) + chunks[2] + overlap[2] 
+                    if z < 0:
+                        z = 0
+                    if z_delt > arraysize[2] -1:
+                        z_delt = arraysize[2]
+                    data = darr[x:x_delt, y:y_delt, z:z_delt]
+                    print(data.shape)
+                    dest = '-' + str(i) +'-' + str(j) + '-' + str(k)
+                    t = wcs_obj[x:x_delt, y:y_delt, z:z_delt]
+                    header.update(t.to_header())
+                    hdu = fits.PrimaryHDU(data=data.compute(), header=header)
+                    hdu.writeto(destination.name + dest + "fits")
+                    fpaths.append(destination.name + dest + "fits")
+    return fpaths
+#def array2disk(array, metadata, outputtype, destination):
+
+#def tmpfitsfile2disk(fitsfile, outputtype, destination):
+
+#def tmpfitsfile2array(fitsfile, arraytype, arraysize, chunks=None):
+
+#def makezarrmetadata(fitsheader):
+
+#def makefitsmetadata(fitsheader, location):
+
+#makehdf5metadata
+
+#makexradiometadata

fits2tmp-0.4.0/LICENSE

@@ -1,28 +0,0 @@
-BSD 3-Clause License
-
-Copyright (c) 2024, Verity Allan
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-1. Redistributions of source code must retain the above copyright notice, this
-   list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright notice,
-   this list of conditions and the following disclaimer in the documentation
-   and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-   contributors may be used to endorse or promote products derived from
-   this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

fits2tmp-0.4.0/PKG-INFO

@@ -1,93 +0,0 @@
-Metadata-Version: 2.3
-Name: fits2tmp
-Version: 0.4.0
-Summary: A way of loading FITS files into temporary filesystems to make them available in memory.
-License: BSD-3-Clause
-Author: Verity Allan
-Author-email: vla22@cam.ac.uk
-Requires-Python: >=3.10,<4.0
-Classifier: License :: OSI Approved :: BSD License
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Classifier: Programming Language :: Python :: 3.13
-Requires-Dist: aiohttp (>=3.13.3,<4.0.0)
-Requires-Dist: astropy (>=6.1.4,<7.0.0)
-Requires-Dist: dask[distributed] (>=2025.1.0,<2026.0.0)
-Requires-Dist: fsspec (>=2026.2.0,<2027.0.0)
-Requires-Dist: h5py (>=3.12.1,<4.0.0)
-Requires-Dist: numpy (>=2.1.2,<3.0.0)
-Requires-Dist: pytest (>=8.3.3,<9.0.0)
-Requires-Dist: requests (>=2.32.5,<3.0.0)
-Requires-Dist: s3fs (>=2026.2.0,<2027.0.0)
-Requires-Dist: zarr (>=2.18.3,<3.0.0)
-Project-URL: Repository, https://gitlab.developers.cam.ac.uk/phy/ra/vla22-phd/fits2tmp
-Description-Content-Type: text/markdown
-
-# fits2tmp 
-
-This project allows you to take a large FITS file and render it as a set of smaller FITS files stored in-memory in a temporary directory, with appropriately updated headers.
-The program returns a list of the location of these files, which can then be processed, by other programs,perhaps by a parallel processing framework such as Dask, or by using Python's `subprocess` module to allow feeding those files to a non-Python program with a command line interface (CLI).
-
-The program also allows you to turn a zarr file containing your FITS data into small FITS files, provided you have extracted the FITS file header to a text file.
-
-This can also be done for hdf5 files (again assuming FITS file header data is saved to a text file). Note that if you attempt to do this with Dask running on multiple nodes, it will currently fail, because of a bug in Dask.
-
-Utilities in the program can be used to write headers out to or read them from a text file.
-
-## TODO:
-
-- Add a utility to convert FITS files to zarr, including saving the header out to a text file.
-- Add hdf5 support, using more native hdf5 metadata storage solutions.
-- Add xradio/xarray support.
-
-## Installing fits2tmp
-
-The program is available from pypi so can be installed with pip: `pip install fits2tmp`.
-
-## Example Usage
-
-```
-import fits2tmp as f2t
-
-a = f2t.fitstmpfiles("fits", "/path/to/input/data", "/path/to/temporary/filesystem", "path/to/header/", (100, 100, 100), chunks=(20,20,20))
-```
-fitstmpfiles takes a large input data set in one of the supported formats, and returns smaller FITS files to a specified directory.
-
-The parameters for fitstmpfiles are:
-
-- filetype: a string describing the filetype; currently supports "fits" and "zarr" fully; "hdf5" only supported on single node runs.
-- inputpath: path to the input data
-- destination: path to destination directory. It is intended that temporary directories are used, to try to keep the data in-memory, rather than doing potentially expensive disk writes; however, if you wish to write to more permanent storage, that is possible. You will probably want to clean up manually once the run is done, though.
-- headerloc: defaults to `None`. If loading from a FITS file, this isn't required; if you are loading from zarr or hdf5, you need to provide the path to a text file containing the FITS headers. It is recommended that you use the fits2tmp utilities to prepare the header file.
-- chunks: defaults to `None`. You will want to set this to some value that makes sense for the resources on which you're processing the data. Dask recommend that you have at least 10 times as much RAM as the size of the chunk you're processing. You will also want to consider how much overlap you are using when setting the chunk size. If you are particularly concerned about file loading performance, then consider setting the chunk size to a multiple of the block size used in the storage. 
-- overlap: defaults to `None`. You will want to set this to some size that makes sense for the data processing you're doing. You probably want to set this to approximately the size of the largest Gaussian kernel you plan to use to detect objects/features.
-
-## Utilities
-
-These small methods are provided to make it easy to write out a FITS header to a file, to allow provision of FITS metadata when using other storage solutions.
-To create a header from an existing FITS file:
-
-```
-from astropy.io import fits
-import fits2tmp as f2t
-
-with fits.open('path/to/fits/file') as f:
-    header = f[0].header
-    d = f2t.header_dict(header)
-    f2t.dict2file(d, 'path/to/where/you/save/header')
-```
-
-Note that this process strips most of the history from the header; if you wish to update the history, use some of the astropy.io.fits tools to accomplish this. This tool is purely to provide a minimal amount of metadata to allow storing FITS data in zarr and similar file formats, and also to allow processing of sections of FITS files where it's not desirable to have all processes reading from the same file in storage.
-
-To create a header from text file containing a json dump of a dictionary created from a FITS header:
-
-```
-import fits2tmp as ft2
-
-loaded_header_dict = f2t.file2dict('/path/to/header/txt/file')
-header = dict2header(loaded_header_dict)
-```
-This leaves you with a valid FITS header
-

fits2tmp-0.4.0/README.md

@@ -1,65 +0,0 @@
-# fits2tmp 
-
-This project allows you to take a large FITS file and render it as a set of smaller FITS files stored in-memory in a temporary directory, with appropriately updated headers.
-The program returns a list of the location of these files, which can then be processed, by other programs,perhaps by a parallel processing framework such as Dask, or by using Python's `subprocess` module to allow feeding those files to a non-Python program with a command line interface (CLI).
-
-The program also allows you to turn a zarr file containing your FITS data into small FITS files, provided you have extracted the FITS file header to a text file.
-
-This can also be done for hdf5 files (again assuming FITS file header data is saved to a text file). Note that if you attempt to do this with Dask running on multiple nodes, it will currently fail, because of a bug in Dask.
-
-Utilities in the program can be used to write headers out to or read them from a text file.
-
-## TODO:
-
-- Add a utility to convert FITS files to zarr, including saving the header out to a text file.
-- Add hdf5 support, using more native hdf5 metadata storage solutions.
-- Add xradio/xarray support.
-
-## Installing fits2tmp
-
-The program is available from pypi so can be installed with pip: `pip install fits2tmp`.
-
-## Example Usage
-
-```
-import fits2tmp as f2t
-
-a = f2t.fitstmpfiles("fits", "/path/to/input/data", "/path/to/temporary/filesystem", "path/to/header/", (100, 100, 100), chunks=(20,20,20))
-```
-fitstmpfiles takes a large input data set in one of the supported formats, and returns smaller FITS files to a specified directory.
-
-The parameters for fitstmpfiles are:
-
-- filetype: a string describing the filetype; currently supports "fits" and "zarr" fully; "hdf5" only supported on single node runs.
-- inputpath: path to the input data
-- destination: path to destination directory. It is intended that temporary directories are used, to try to keep the data in-memory, rather than doing potentially expensive disk writes; however, if you wish to write to more permanent storage, that is possible. You will probably want to clean up manually once the run is done, though.
-- headerloc: defaults to `None`. If loading from a FITS file, this isn't required; if you are loading from zarr or hdf5, you need to provide the path to a text file containing the FITS headers. It is recommended that you use the fits2tmp utilities to prepare the header file.
-- chunks: defaults to `None`. You will want to set this to some value that makes sense for the resources on which you're processing the data. Dask recommend that you have at least 10 times as much RAM as the size of the chunk you're processing. You will also want to consider how much overlap you are using when setting the chunk size. If you are particularly concerned about file loading performance, then consider setting the chunk size to a multiple of the block size used in the storage. 
-- overlap: defaults to `None`. You will want to set this to some size that makes sense for the data processing you're doing. You probably want to set this to approximately the size of the largest Gaussian kernel you plan to use to detect objects/features.
-
-## Utilities
-
-These small methods are provided to make it easy to write out a FITS header to a file, to allow provision of FITS metadata when using other storage solutions.
-To create a header from an existing FITS file:
-
-```
-from astropy.io import fits
-import fits2tmp as f2t
-
-with fits.open('path/to/fits/file') as f:
-    header = f[0].header
-    d = f2t.header_dict(header)
-    f2t.dict2file(d, 'path/to/where/you/save/header')
-```
-
-Note that this process strips most of the history from the header; if you wish to update the history, use some of the astropy.io.fits tools to accomplish this. This tool is purely to provide a minimal amount of metadata to allow storing FITS data in zarr and similar file formats, and also to allow processing of sections of FITS files where it's not desirable to have all processes reading from the same file in storage.
-
-To create a header from text file containing a json dump of a dictionary created from a FITS header:
-
-```
-import fits2tmp as ft2
-
-loaded_header_dict = f2t.file2dict('/path/to/header/txt/file')
-header = dict2header(loaded_header_dict)
-```
-This leaves you with a valid FITS header

fits2tmp-0.4.0/pyproject.toml

@@ -1,31 +0,0 @@
-[tool.poetry]
-name = "fits2tmp"
-version = "0.4.0"
-description = "A way of loading FITS files into temporary filesystems to make them available in memory."
-license = "BSD-3-Clause"
-authors = ["Verity Allan <vla22@cam.ac.uk>"]
-readme = "README.md"
-repository = "https://gitlab.developers.cam.ac.uk/phy/ra/vla22-phd/fits2tmp"
-
-[tool.poetry.dependencies]
-python = "^3.10"
-dask = {extras = ["distributed"], version = "^2025.1.0"}
-astropy = "^6.1.4"
-h5py = "^3.12.1"
-numpy = "^2.1.2"
-pytest = "^8.3.3"
-zarr = "^2.18.3"
-fsspec = "^2026.2.0"
-requests = "^2.32.5"
-aiohttp = "^3.13.3"
-s3fs = "^2026.2.0"
-
-[tool.pytest.ini_options]
-addopts = [
-    "--import-mode=importlib",
-]
-pythonpath = "src/fits2tmp/"
-
-[build-system]
-requires = ["poetry-core"]
-build-backend = "poetry.core.masonry.api"

fits2tmp-0.4.0/src/fits2tmp/fits2tmp.py

@@ -1,192 +0,0 @@
-import h5py
-import zarr
-import json
-from astropy.io import fits
-from astropy import wcs
-import dask.array as da
-from dask import delayed
-from dask.distributed import Lock
-import numpy as np
-import os
-import dask 
-import tempfile
-
-def arr_size_check(source_arr, desired_arr: tuple) -> bool:
-    if len(source_arr.shape) < len(desired_arr):
-        raise ValueError("You've asked for an array with more dimensions than the array stored")
-    else:
-        for i in range(min(len(source_arr.shape), len(desired_arr))):
-            # using min here allows us to avoid an IndexError when we have more dimensions in the source array
-            j = source_arr.shape[i] - desired_arr[i]
-            if j < 0:
-                raise IndexError("Desired array size is larger than the stored array in at least position %s: source array has dimensions %s, and you have requested $s", i, source_arr.shape, desired_arr)
-        return True
-
-#def fits_add_header(orig_header, sliced_data) -> fits.Header:
-#    coords = wcs.WCS(orig_header)
-#    new_header = orig_header
-#    new_header.update(coords[data_sli].to_header())
-#    return new_header
-
-def header_dict(header) -> dict:
-    d = {}
-    for i in header.cards:
-        d[i.keyword] = i.value
-    return d
-
-def dict2header(input_dict: dict) -> fits.Header:
-    nh = fits.Header(cards=input_dict)
-    return nh
-
-def dict2file(input_dict: dict, filepath):
-    with open(filepath, 'w') as f:
-        f.write(json.dumps(input_dict))
-
-def file2dict(filepath) -> dict:
-    with open(filepath, 'r') as j:
-        contents = json.loads(j.read())
-    return contents
-filepath = []
-
-def disk2dah(filetype, inputpath, destination, headerloc, arraysize, chunks=None, use_fsspec=False, storage_opts=None): 
-    # TODO: ADD code to handle arraysize differing from size of array in file
-    # maybe extend arr_size_check()?
-    # TODO make this support arrays that have other than 3 dims
-    destination = tempfile.TemporaryDirectory(dir=destination, delete=False)
-    match filetype:
-        case "fits":
-            with fits.open(inputpath, use_fsspec=use_fsspec, fsspec_kwargs=storage_opts) as f:
-                match len(arraysize):
-                    case 1:
-                        if len(arraysize) == len(f[0].data.shape):
-                            d = da.from_array(f[0].data[0:arraysize[0]], chunks=chunks)
-                        elif len(arraysize) == len(f[0].data.shape) - 1:
-                            d = da.from_array(f[0].data[0:arraysize[0],0], chunks=chunks)
-                        elif len(arraysize) == len(f[0].data.shape) -2:
-                            d = da.from_array(f[0].data[0:arraysize[0],0,0], chunks=chunks)
-                        elif len(arraysize) == len(f[0].data.shape) -3:
-                            d = da.from_array(f[0].data[0:arraysize[0],0,0,0], chunks=chunks)
-                        else:
-                            raise ValueError("You've asked to load an array with more than 4 dimensions, which is not supported.")
-                                                
-                    case 2:
-                        if len(arraysize) == len(f[0].data.shape):
-                            d = da.from_array(f[0].data[0:arraysize[0], 0:arraysize[1]], chunks=chunks)
-                        elif len(arraysize) == len(f[0].data.shape) - 1:
-                            d = da.from_array(f[0].data[0:arraysize[0],0:arraysize[1], 0], chunks=chunks)
-                        elif len(arraysize) == len(f[0].data.shape) -2:
-                            d = da.from_array(f[0].data[0:arraysize[0],0:arraysize[1],0,0], chunks=chunks)
-                        else:
-                            raise ValueError("You've asked to load an array with more than 4 dimensions, which is not supported.")
-                    case 3:
-                        if len(arraysize) == len(f[0].data.shape):
-                            d = da.from_array(f[0].data[0:arraysize[0], 0:arraysize[1], 0:arraysize[2]], chunks=chunks)
-                        elif len(arraysize) == len(f[0].data.shape) - 1:
-                            d = da.from_array(f[0].data[0:arraysize[0],0:arrasize[1], 0:arraysize[2],0], chunks=chunks)
-                        else:
-                            raise ValueError("You've asked to load an array with more than 4 dimensions, which is not supported.")
-                    case 4:
-                        d = da.from_array(f[0].data[0:arraysize[0], 0:arraysize[1], 0:arraysize[2], 0:arraysize[3]], chunks=chunks)
-                n_header = dict2header(header_dict(f[0].header))
-                wcs_obj = wcs.WCS(n_header)
-        case "zarr":
-            if use_fsspec == True:
-                d = da.from_zarr(inputpath, storage_options=storage_opts, chunks=chunks)[0:arraysize[0],0:arraysize[1],0:arraysize[2]]
-                print(type(d))
-            else:
-                d = da.from_zarr(inputpath, chunks=chunks)[0:arraysize[0],0:arraysize[1],0:arraysize[2]]
-            n_header = dict2header(file2dict(headerloc))
-            wcs_obj = wcs.WCS(n_header)
-        case "hdf5":
-            with h5py.File(filepath, "r") as f:
-            #note this is hardcoded for the particular hdf5 files tested - needs parameterisation
-                d = da.from_array(f["array"][arraysize], chunks=chunks)
-                n_header = dict2header(file2dict(inputpath, headerloc))
-                wcs_obj = wcs.WCS(n_header)
-
-        case "xradio":
-            weeble
-        case _:
-            raise ValueError("The specified file type is not supported by fits2tmp.")
-    return [d, destination, n_header, wcs_obj]
-
-def fitstmpfiles(filetype, inputpath, destination, arraysize, headerloc=None, chunks=None, overlap=None):
-    inputarray = disk2dah(filetype, inputpath, destination, headerloc, arraysize, chunks)
-    if overlap == None:
-        overlap = [0,0,0]
-    darr = inputarray[0]
-    destination = inputarray[1]
-    header = inputarray[2]
-    wcs_obj = inputarray[3]
-    # why am I pushing this array to delayed? ans: to parallelise construction of the list
-    #dd = darr.to_delayed()
-    fpaths = []
-    data = []
-    for i in range(len(darr.chunks[0])):
-        x = i * chunks[0] - overlap[0] 
-        x_delt = (i* chunks[0]) + chunks[0] + overlap[0] 
-        # range check - when asking for overlap, we curtail at the edges of the array
-        if x < 0:
-            x = 0
-        if x_delt > arraysize[0] -1:
-            x_delt = arraysize[0]
-        if len(darr.shape) == 1:
-            #can't think of why we want to support 1D array stuff, but it's probably OK to do this, though the headers will end up not very useful, probably
-            dest = str(i)
-            data = darr[x:x_delt]
-            t = wcs_obj[x:x_delt]
-            header.update(t.to_header())
-            hdu = fits.PrimaryHDU(data=data, header=header)
-            hdu.writeto(destination.name + dest + "fits")
-            fpaths.append(destination.name + dest + "fits")
-        else:
-            for j in range(len(darr.chunks[1])):
-                y = j * chunks[1] - overlap[1]
-                y_delt = (j * chunks[1]) + chunks[1] + overlap[1] 
-                if y < 0:
-                    y = 0
-                if y_delt > arraysize[1] -1:
-                    y_delt = arraysize[1]
-                if len(darr.shape) == 2:
-                    dest = str(i) + str(j)
-                    data = darr[x:x_delt, y:y_delt]
-                    t = wcs_obj[x:x_delt, y:y_delt]
-                    header.update(t.to_header())
-                    hdu = fits.PrimaryHDU(data=data, header=header)
-                    hdu.writeto(destination.name + dest + "fits")
-                    fpaths.append(destination.name + dest + "fits")
-                elif len(darr.shape) > 2:
-                    for k in range(len(darr.chunks[2])):
-                        z = k * chunks[2] - overlap[2]
-                        z_delt = (k * chunks[2]) + chunks[2] + overlap[2] 
-                        if z < 0:
-                            z = 0
-                        if z_delt > arraysize[2] -1:
-                            z_delt = arraysize[2]
-                        dest = '-' + str(i) +'-' + str(j) + '-' + str(k)
-                        if len(darr.shape) == 3:
-                            data = darr[x:x_delt, y:y_delt, z:z_delt]
-                            t = wcs_obj[x:x_delt, y:y_delt, z:z_delt]
-                            header.update(t.to_header())
-                        elif len(darr.shape) == 4:
-                            # this assumes we're dealing with radio astronomy data, so we just select all indices in the 4th axis, which is usually polarisation information
-                            data = darr[x:x_delt, y:y_delt, z:z_delt, :]
-                            t = wcs_obj[x:x_delt, y:y_delt, z:z_delt, :]
-                            header.update(t.to_header())
-                        hdu = fits.PrimaryHDU(data=data, header=header)
-                        hdu.writeto(destination.name + dest + "fits")
-                        fpaths.append(destination.name + dest + "fits")
-    return fpaths
-#def array2disk(array, metadata, outputtype, destination):
-
-#def tmpfitsfile2disk(fitsfile, outputtype, destination):
-
-#def tmpfitsfile2array(fitsfile, arraytype, arraysize, chunks=None):
-
-#def makezarrmetadata(fitsheader):
-
-#def makefitsmetadata(fitsheader, location):
-
-#makehdf5metadata
-
-#makexradiometadata