xradio 0.0.24__py3-none-any.whl → 0.0.25__py3-none-any.whl

xradio/image/_util/_zarr/zarr_low_level.py

@@ -109,19 +109,55 @@ def write_binary_blob_to_disk(arr, file_path, compressor):
     Returns:
     - None
     """
+    import graphviper.utils.logger as logger
     # Encode the NumPy array using the codec
+    logger.debug('1. Before compressor ' + file_path)
     compressed_arr = compressor.encode(np.ascontiguousarray(arr))
 
+    logger.debug('2. Before makedir')
     # Ensure the directory exists before saving the file
     os.makedirs(os.path.dirname(file_path), exist_ok=True)
 
-    # Save the compressed array to disk
-    with open(file_path, "wb") as file:
-        file.write(compressed_arr)
+    arr_len = len(compressed_arr)
+    logger.debug('3. Before write the len is: ' + str(arr_len))
+    #Save the compressed array to disk
+    # with open(file_path, "wb") as file:
+    #     file.write(compressed_arr)
+    
+    logger.debug('4. Using new writer: ' + str(arr_len))
+    write_to_lustre_chunked(file_path, compressed_arr)
+        
+    # /.lustre/aoc/sciops/pford/CHILES/cube_image/uid___A002_Xee7674_X2844_Cube_3.img.zarr/SKY/0.0.110.0.0
+    # 348192501 bytes
+    # 332.0622453689575 M
+    
+    # from io import BufferedWriter
+    # # Calculate buffer size based on compressed_arr size (adjust multiplier)
+    # buffer_size = min(len(compressed_arr), 1024 * 1024 * 4)  # Max 4 MB buffer
+    # with BufferedWriter(open(file_path, "wb"), buffer_size) as f:
+    #     f.write(compressed_arr)
+    #     f.flush()  # Ensure data gets written to disk 
+    
+    
+    logger.debug('4. Write completed')
 
     # print(f"Compressed array saved to {file_path}")
 
 
+def write_to_lustre_chunked(file_path, compressed_arr, chunk_size=1024 * 1024 * 128):  # 128 MiB chunks
+  """
+  Writes compressed data to a Lustre file path with chunking.
+
+  Args:
+      file_path: Path to the file for writing.
+      compressed_arr: Compressed data array to write.
+      chunk_size: Size of each data chunk in bytes (default: 128 MiB).
+  """
+  with open(file_path, "wb") as f:
+    for i in range(0, len(compressed_arr), chunk_size):
+      chunk = compressed_arr[i:i + chunk_size]
+      f.write(chunk)
+
 def read_binary_blob_from_disk(file_path, compressor, dtype=np.float64):
     """
     Read a compressed binary blob from disk and decode it using Blosc.

xradio/vis/_processing_set.py

@@ -43,7 +43,10 @@ class processing_set(dict):
             "start_frequency": [],
             "end_frequency": [],
             "shape": [],
+            "field_coords": []
         }
+        from astropy.coordinates import SkyCoord
+        import astropy.units as u
         for key, value in self.items():
             summary_data["name"].append(key)
             summary_data["ddi"].append(value.attrs["ddi"])
@@ -65,6 +68,12 @@ class processing_set(dict):
             )
             summary_data["start_frequency"].append(value["frequency"].values[0])
             summary_data["end_frequency"].append(value["frequency"].values[-1])
+
+            ra_dec_rad = value[data_name].attrs["field_info"]['phase_direction']['data']
+            frame = value[data_name].attrs["field_info"]['phase_direction']['attrs']['frame'].lower()
+            coord = SkyCoord(ra=ra_dec_rad[0]*u.rad, dec=ra_dec_rad[1]*u.rad, frame=frame)
+
+            summary_data["field_coords"].append([frame, coord.ra.to_string(unit=u.hour), coord.dec.to_string(unit=u.deg)])
         summary_df = pd.DataFrame(summary_data)
         return summary_df
 

{xradio-0.0.24.dist-info → xradio-0.0.25.dist-info}/METADATA

@@ -1,7 +1,7 @@
 Metadata-Version: 2.1
 Name: xradio
-Version: 0.0.24
-Summary: Xarray Radio Astronomy Data IO 
+Version: 0.0.25
+Summary: Xarray Radio Astronomy Data IO
 Author-email: Jan-Willem Steeb <jsteeb@nrao.edu>
 License: BSD 3-Clause License
         
@@ -32,7 +32,7 @@ License: BSD 3-Clause License
         OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
         OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
         
-Requires-Python: <3.12,>=3.8
+Requires-Python: <3.12,>=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE.txt
 Requires-Dist: astropy

{xradio-0.0.24.dist-info → xradio-0.0.25.dist-info}/RECORD

@@ -20,7 +20,7 @@ xradio/image/_util/_fits/xds_from_fits.py,sha256=SIYyl4dAoCdiB4CSv0_wV_hY7jhLZZE
 xradio/image/_util/_zarr/common.py,sha256=apMX_bF4Hr3pFGjnDFpp36KgmhTYAPBZquNkjBHrsXk,307
 xradio/image/_util/_zarr/xds_from_zarr.py,sha256=hz6lHlpybfr_r8pn_uObDHOFmN5h75F11bkBv8KCuP0,3192
 xradio/image/_util/_zarr/xds_to_zarr.py,sha256=wogXbwX8n3Sl9PHoc3_Y_LBowQsQ-94HZQFZ5NcxUZA,1624
-xradio/image/_util/_zarr/zarr_low_level.py,sha256=DuAD1kVUbHjOxVH9jRagk9GdPA5uH2fIMzA0djxM6Sc,9537
+xradio/image/_util/_zarr/zarr_low_level.py,sha256=5D8Vu8QZD8CpKSPMkxHhnKpdk3vhUStW3kQFHUQUQns,10984
 xradio/schema/__init__.py,sha256=UpejQegOaCLrsbcR4MLudR0RxeE0sN3zxFXM8rzyJPo,444
 xradio/schema/bases.py,sha256=vcW47jZWpJ0mJdni7eFVY7zJoper2sy2VjX8LE3pUqc,150
 xradio/schema/check.py,sha256=3u79hRL3pGF6dvQE0LF21nGdAVnRXWwgnbMmStGBSSA,16310
@@ -28,7 +28,7 @@ xradio/schema/dataclass.py,sha256=vkc2cqLjGV5QN8j70GbBaNfslT_KLWmebsPGeBEuGcs,88
 xradio/schema/metamodel.py,sha256=RHrihyaetinu7_lGTTZ31Rlv-_Db_EgQCXzk56H004o,3476
 xradio/schema/typing.py,sha256=coF3LuKOlCUJGKTUUH81EcjePZ86koOYzm8qzsAw-HU,9983
 xradio/vis/__init__.py,sha256=AV2WG26NzFB1LEEtFaq1ULQKz9VnluEAjg0Qb5Ju7m8,358
-xradio/vis/_processing_set.py,sha256=3cm-WAuBjzX0_XPUjSR1i2i5_PeXiGwW12XioVji2Wk,3666
+xradio/vis/_processing_set.py,sha256=zYCswmWjsRN_8M61IkbMo6zpI-4ABMWpKZkL8f20KGo,4187
 xradio/vis/convert_msv2_to_processing_set.py,sha256=7vTjqtWFEBOySnLVoadceKCA4VLgOig7eCs1dxrdYQA,3966
 xradio/vis/load_processing_set.py,sha256=DsKjizvcJxXNn5BuXH2fye-dzMoB-BDK2f-8jsvK2QU,4776
 xradio/vis/model.py,sha256=uBjvvhYEY1p-3H3NStrt1ZKMQACLGLo93OiEBvDVId8,17083
@@ -64,8 +64,8 @@ xradio/vis/_vis_utils/_utils/xds_helper.py,sha256=UhtAZV5DyYzVVBkXzwDAOz6TICxotQ
 xradio/vis/_vis_utils/_zarr/encoding.py,sha256=GENIlThV6a9CUCL6gIGlu9c6NR3OFWNos6mpxZjMwDc,536
 xradio/vis/_vis_utils/_zarr/read.py,sha256=ikNGlOdHuZ_cgWpPAZ4iHzeLdU44I0iBLcqSEiM_hCk,7111
 xradio/vis/_vis_utils/_zarr/write.py,sha256=exvrqNVnVKk6LzbDPm_fm142YzX-7lHGqklXTQB9wh0,8864
-xradio-0.0.24.dist-info/LICENSE.txt,sha256=dvACd-5O67yjSZlnEKcWmu3DqwzBtbC922iPv0KOeAw,1516
-xradio-0.0.24.dist-info/METADATA,sha256=yJW0v_5BpEInE5IVWDbELOpAHGGzgT0a0PmrbLnWYZ8,4070
-xradio-0.0.24.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-xradio-0.0.24.dist-info/top_level.txt,sha256=dQu27fGBZJ2Yk-gW5XeD-dZ76Xa4Xcvk60Vz-dwXp7k,7
-xradio-0.0.24.dist-info/RECORD,,
+xradio-0.0.25.dist-info/LICENSE.txt,sha256=dvACd-5O67yjSZlnEKcWmu3DqwzBtbC922iPv0KOeAw,1516
+xradio-0.0.25.dist-info/METADATA,sha256=c_uoGScVmVEzC-iSPJ0KczSHcHN0Wt6ZRRD4k7qVQrw,4069
+xradio-0.0.25.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+xradio-0.0.25.dist-info/top_level.txt,sha256=dQu27fGBZJ2Yk-gW5XeD-dZ76Xa4Xcvk60Vz-dwXp7k,7
+xradio-0.0.25.dist-info/RECORD,,