fits2tmp 0.1.9__tar.gz → 0.3.0__tar.gz

fits2tmp-0.3.0/PKG-INFO

@@ -0,0 +1,50 @@
+Metadata-Version: 2.3
+Name: fits2tmp
+Version: 0.3.0
+Summary: A way of loading FITS files into temporary filesystems to make them available in memory.
+License: BSD-3-Clause
+Author: Verity Allan
+Author-email: vla22@cam.ac.uk
+Requires-Python: >=3.10,<4.0
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: astropy (>=6.1.4,<7.0.0)
+Requires-Dist: dask[distributed] (>=2025.1.0,<2026.0.0)
+Requires-Dist: h5py (>=3.12.1,<4.0.0)
+Requires-Dist: numpy (>=2.1.2,<3.0.0)
+Requires-Dist: pytest (>=8.3.3,<9.0.0)
+Requires-Dist: zarr (>=2.18.3,<3.0.0)
+Project-URL: Repository, https://gitlab.developers.cam.ac.uk/phy/ra/vla22-phd/fits2tmp
+Description-Content-Type: text/markdown
+
+# fits2tmp 
+
+This project allows you to take a large FITS file and render it as a set of smaller FITS files stored in-memory in a temporary directory, with appropriately updated headers.
+The program returns a list of the location of these files, which can then be processed by a parallel processing framework such as dask.
+
+The program also allows you to turn a zarr file containing your FITS data into small FITS files, provided you have extracted the FITS file header to a text file.
+Utilities in the program can be used to write headers out to or read them from a text file.
+
+## TODO:
+
+- Add a utility to convert FITS files to zarr, including saving the header out to a text file.
+- Add hdf5 support using a similar strategy to FITS.
+- Add xradio support.
+
+## Installing fits2tmp
+
+The program is available from pypi so can be installed with pip: `pip install fits2tmp`.
+
+## Example Usage
+
+```
+import fits2tmp as f2t
+
+a = fakefits2tmpfiles("fits", "/path/to/input/data", "/path/to/temporary/filesystem", "path/to/header/", (100, 100, 100), chunks=(20,20,20))
+```
+
+

fits2tmp-0.3.0/README.md

@@ -0,0 +1,26 @@
+# fits2tmp 
+
+This project allows you to take a large FITS file and render it as a set of smaller FITS files stored in-memory in a temporary directory, with appropriately updated headers.
+The program returns a list of the location of these files, which can then be processed by a parallel processing framework such as dask.
+
+The program also allows you to turn a zarr file containing your FITS data into small FITS files, provided you have extracted the FITS file header to a text file.
+Utilities in the program can be used to write headers out to or read them from a text file.
+
+## TODO:
+
+- Add a utility to convert FITS files to zarr, including saving the header out to a text file.
+- Add hdf5 support using a similar strategy to FITS.
+- Add xradio support.
+
+## Installing fits2tmp
+
+The program is available from pypi so can be installed with pip: `pip install fits2tmp`.
+
+## Example Usage
+
+```
+import fits2tmp as f2t
+
+a = fakefits2tmpfiles("fits", "/path/to/input/data", "/path/to/temporary/filesystem", "path/to/header/", (100, 100, 100), chunks=(20,20,20))
+```
+

{fits2tmp-0.1.9 → fits2tmp-0.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "fits2tmp"
-version = "0.1.9"
+version = "0.3.0"
 description = "A way of loading FITS files into temporary filesystems to make them available in memory."
 license = "BSD-3-Clause"
 authors = ["Verity Allan <vla22@cam.ac.uk>"]

{fits2tmp-0.1.9 → fits2tmp-0.3.0}/src/fits2tmp/fits2tmp.py

@@ -79,50 +79,85 @@ def fits2tmpfiles(inputpath, destination, chunks):
 
     return filepath
 
-def disk2fakefits(filetype, inputpath, destination, arraysize, chunks=None):
+def disk2dah(filetype, inputpath, destination, headerloc, arraysize, chunks=None):
+    # TODO: ADD code to handle arraysize differing from size of array in file
+    # maybe extend arr_size_check()?
+    # TODO make this support arrays that have other than 3 dims
+    destination = tempfile.TemporaryDirectory(dir=destination, delete=False)
     match filetype:
         case "fits":
             f = fits.open(inputpath)
-            d = da.from_array(f[0].data, chunks=chunks)
-            destination = tempfile.TemporaryDirectory(dir=destination, delete=False)
+            d = da.from_array(f[0].data[0:arraysize[0], 0:arraysize[1], 0:arraysize[2]], chunks=chunks)
             n_header = dict2header(header_dict(f[0].header))
             wcs_obj = wcs.WCS(n_header)
         case "zarr":
-            bar
+            d = da.from_zarr(inputpath, chunks=chunks)[arraysize]
+            n_header = dict2header(file2dict(inputpath, headerloc))
+            wcs_obj = wcs.WCS(n_header)
         case "hdf5":
-            baz
+            f = h5py.File(filepath, "r")
+            #note this is hardcoded for the particular hdf5 files tested - needs parameterisation
+            d = da.from_array(f["array"][arraysize], chunks=chunks)
+            n_header = dict2header(file2dict(inputpath, headerloc))
+            wcs_obj = wcs.WCS(n_header)
+
         case "xradio":
             weeble
         case _:
             raise ValueError("The specified file type is not supported by fits2tmp.")
-    return d, destination, n_header, wcs_obj
+    return [d, destination, n_header, wcs_obj]
 
-def fakefits2tmpfiles(filetype, inputpath, destination, arraysize, chunks=None):
-    darr, tempdest, n_header, wcs_obj = disk2fakefits(filetype, inputpath, destination, arraysize, chunks)
+def fakefits2tmpfiles(filetype, inputpath, destination, arraysize, headerloc=None, chunks=None, overlap=None):
+    inputarray = disk2dah(filetype, inputpath, destination, headerloc, arraysize, chunks)
+    if overlap == None:
+        overlap = [0,0,0]
+    darr = inputarray[0]
     print(darr.shape)
-    dd = darr.to_delayed()
+    print(darr)
+    destination = inputarray[1]
+    header = inputarray[2]
+    print(header)
+    wcs_obj = inputarray[3]
+    print(len(darr.chunks))
+    # why am I pushing this array to delayed? ans: to parallelise construction of the list
+    #dd = darr.to_delayed()
     fpaths = []
-    for i in range(dd.shape[0]):
-        x = i * chunks[0] 
-        x_delt = x + chunks[0] - 1
-        for j in range(dd.shape[1]):
-            y = j * chunks[1]
-            y_delt = y + chunks[1] - 1
-            if len(dd.shape) == 2:
+    for i in range(len(darr.chunks[0])):
+        x = i * chunks[0] - overlap[0] 
+        x_delt = (i* chunks[0]) + chunks[0] + overlap[0] 
+        # range check - when asking for overlap, we curtail at the edges of the array
+        if x < 0:
+            x = 0
+        if x_delt > arraysize[0] -1:
+            x_delt = arraysize[0]
+        print(x, x_delt)
+        for j in range(len(darr.chunks[1])):
+            y = j * chunks[1] - overlap[1]
+            y_delt = (j * chunks[1]) + chunks[1] + overlap[1] 
+            if y < 0:
+                y = 0
+            if y_delt > arraysize[1] -1:
+                y_delt = arraysize[1]
+            if len(darr.shape) == 2:
                 dest = str(i) + str(j)
                 data = dd[i][j]
                 wcs_coords = wcs_obj.slice((slice(x,x_delt), slice(y,y_delt)))
-            elif len(dd.shape) > 2:
-                for k in range(dd.shape[2]):
-                    data = dd[i][j][k]
-                    z = k * chunks[2]
-                    z_delt = z + chunks[2] - 1
-                    dest = str(i) + str(j) + str(k)
-                    t =wcs_obj[x:x_delt, y:y_delt, z:z_delt]
-                    n_header.update(t.to_header())
-                    hdu = fits.PrimaryHDU(data=data.compute(), header=n_header)
-                    hdu.writeto(tempdest.name + dest + "fits")
-                    fpaths.append(tempdest.name + dest + "fits")
+            elif len(darr.shape) > 2:
+                for k in range(len(darr.chunks[2])):
+                    z = k * chunks[2] - overlap[2]
+                    z_delt = (k * chunks[2]) + chunks[2] + overlap[2] 
+                    if z < 0:
+                        z = 0
+                    if z_delt > arraysize[2] -1:
+                        z_delt = arraysize[2]
+                    data = darr[x:x_delt, y:y_delt, z:z_delt]
+                    print(data.shape)
+                    dest = '-' + str(i) +'-' + str(j) + '-' + str(k)
+                    t = wcs_obj[x:x_delt, y:y_delt, z:z_delt]
+                    header.update(t.to_header())
+                    hdu = fits.PrimaryHDU(data=data.compute(), header=header)
+                    hdu.writeto(destination.name + dest + "fits")
+                    fpaths.append(destination.name + dest + "fits")
     return fpaths
 #def array2disk(array, metadata, outputtype, destination):
 

fits2tmp-0.1.9/PKG-INFO

@@ -1,31 +0,0 @@
-Metadata-Version: 2.1
-Name: fits2tmp
-Version: 0.1.9
-Summary: A way of loading FITS files into temporary filesystems to make them available in memory.
-Home-page: https://gitlab.developers.cam.ac.uk/phy/ra/vla22-phd/fits2tmp
-License: BSD-3-Clause
-Author: Verity Allan
-Author-email: vla22@cam.ac.uk
-Requires-Python: >=3.10,<4.0
-Classifier: License :: OSI Approved :: BSD License
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: astropy (>=6.1.4,<7.0.0)
-Requires-Dist: dask[distributed] (>=2025.1.0,<2026.0.0)
-Requires-Dist: h5py (>=3.12.1,<4.0.0)
-Requires-Dist: numpy (>=2.1.2,<3.0.0)
-Requires-Dist: pytest (>=8.3.3,<9.0.0)
-Requires-Dist: zarr (>=2.18.3,<3.0.0)
-Project-URL: Repository, https://gitlab.developers.cam.ac.uk/phy/ra/vla22-phd/fits2tmp
-Description-Content-Type: text/markdown
-
-# Sample GitLab Project
-
-This sample project shows how a project in GitLab looks for demonstration purposes. It contains issues, merge requests and Markdown files in many branches,
-named and filled with lorem ipsum.
-
-You can look around to get an idea how to structure your project and, when done, you can safely delete this project.
-
-[Learn more about creating GitLab projects.](https://docs.gitlab.com/ee/gitlab-basics/create-project.html)
-

fits2tmp-0.1.9/README.md

@@ -1,8 +0,0 @@
-# Sample GitLab Project
-
-This sample project shows how a project in GitLab looks for demonstration purposes. It contains issues, merge requests and Markdown files in many branches,
-named and filled with lorem ipsum.
-
-You can look around to get an idea how to structure your project and, when done, you can safely delete this project.
-
-[Learn more about creating GitLab projects.](https://docs.gitlab.com/ee/gitlab-basics/create-project.html)