xradio 0.0.28__py3-none-any.whl → 0.0.29__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. xradio/__init__.py +5 -4
  2. xradio/_utils/array.py +90 -0
  3. xradio/_utils/zarr/common.py +48 -3
  4. xradio/image/_util/zarr.py +4 -1
  5. xradio/schema/__init__.py +24 -6
  6. xradio/schema/bases.py +440 -2
  7. xradio/schema/check.py +96 -55
  8. xradio/schema/dataclass.py +123 -27
  9. xradio/schema/metamodel.py +21 -4
  10. xradio/schema/typing.py +33 -18
  11. xradio/vis/__init__.py +5 -2
  12. xradio/vis/_processing_set.py +28 -20
  13. xradio/vis/_vis_utils/_ms/_tables/create_field_and_source_xds.py +710 -0
  14. xradio/vis/_vis_utils/_ms/_tables/load.py +23 -10
  15. xradio/vis/_vis_utils/_ms/_tables/load_main_table.py +145 -64
  16. xradio/vis/_vis_utils/_ms/_tables/read.py +747 -172
  17. xradio/vis/_vis_utils/_ms/_tables/read_main_table.py +173 -44
  18. xradio/vis/_vis_utils/_ms/_tables/read_subtables.py +79 -28
  19. xradio/vis/_vis_utils/_ms/_tables/write.py +102 -45
  20. xradio/vis/_vis_utils/_ms/_tables/write_exp_api.py +127 -65
  21. xradio/vis/_vis_utils/_ms/chunks.py +58 -21
  22. xradio/vis/_vis_utils/_ms/conversion.py +536 -67
  23. xradio/vis/_vis_utils/_ms/descr.py +52 -20
  24. xradio/vis/_vis_utils/_ms/msv2_to_msv4_meta.py +70 -35
  25. xradio/vis/_vis_utils/_ms/msv4_infos.py +0 -59
  26. xradio/vis/_vis_utils/_ms/msv4_sub_xdss.py +76 -9
  27. xradio/vis/_vis_utils/_ms/optimised_functions.py +0 -46
  28. xradio/vis/_vis_utils/_ms/partition_queries.py +308 -119
  29. xradio/vis/_vis_utils/_ms/partitions.py +82 -25
  30. xradio/vis/_vis_utils/_ms/subtables.py +32 -14
  31. xradio/vis/_vis_utils/_utils/partition_attrs.py +30 -11
  32. xradio/vis/_vis_utils/_utils/xds_helper.py +136 -45
  33. xradio/vis/_vis_utils/_zarr/read.py +60 -22
  34. xradio/vis/_vis_utils/_zarr/write.py +83 -9
  35. xradio/vis/_vis_utils/ms.py +48 -29
  36. xradio/vis/_vis_utils/zarr.py +44 -20
  37. xradio/vis/convert_msv2_to_processing_set.py +106 -32
  38. xradio/vis/load_processing_set.py +38 -61
  39. xradio/vis/read_processing_set.py +62 -96
  40. xradio/vis/schema.py +687 -0
  41. xradio/vis/vis_io.py +75 -43
  42. {xradio-0.0.28.dist-info → xradio-0.0.29.dist-info}/LICENSE.txt +6 -1
  43. {xradio-0.0.28.dist-info → xradio-0.0.29.dist-info}/METADATA +10 -5
  44. xradio-0.0.29.dist-info/RECORD +73 -0
  45. {xradio-0.0.28.dist-info → xradio-0.0.29.dist-info}/WHEEL +1 -1
  46. xradio/vis/model.py +0 -497
  47. xradio-0.0.28.dist-info/RECORD +0 -71
  48. {xradio-0.0.28.dist-info → xradio-0.0.29.dist-info}/top_level.txt +0 -0
@@ -17,16 +17,29 @@ def load_col_chunk(
17
17
  """
18
18
  Loads a slice of a col (using casacore getcol(slice))
19
19
 
20
- :param tb_tool: a table/TaQL query open and being used to load columns
21
- :param col: colum to load
22
- :param cshape: shape of the resulting col data chunk
23
- :param tidxs: time axis indices
24
- :param bidxs: baseline axis indices
25
- :param didxs: (effective) data indices, excluding missing baselines
26
- :param d1: indices to load on dimension 1 (None=all, pols or chans)
27
- :param d2: indices to load on dimension 2 (None=all, pols)
28
-
29
- :return: data array loaded directly with casacore getcol/getcolslice
20
+ Parameters
21
+ ----------
22
+ tb_tool : tables.table
23
+ a table/TaQL query open and being used to load columns
24
+ col : str
25
+ colum to load
26
+ cshape : Tuple[int]
27
+ shape of the resulting col data chunk
28
+ tidxs : np.ndarray
29
+ time axis indices
30
+ bidxs : np.ndarray
31
+ baseline axis indices
32
+ didxs : np.ndarray
33
+ effective) data indices, excluding missing baselines
34
+ d1 : Tuple[int, int]
35
+ indices to load on dimension 1 (None=all, pols or chans)
36
+ d2 : Tuple[int, int]
37
+ indices to load on dimension 2 (None=all, pols)
38
+
39
+ Returns
40
+ -------
41
+ np.ndarray
42
+ data array loaded directly with casacore getcol/getcolslice
30
43
  """
31
44
 
32
45
  if (len(cshape) == 2) or (col == "UVW"):
@@ -16,7 +16,7 @@ from xradio.vis._vis_utils._ms._tables.read_main_table import (
16
16
  get_baselines,
17
17
  get_baseline_indices,
18
18
  )
19
- from xradio.vis._vis_utils._ms.optimised_functions import unique_1d
19
+ from xradio._utils.array import unique_1d
20
20
 
21
21
 
22
22
  def load_expanded_main_table_chunk(
@@ -29,12 +29,21 @@ def load_expanded_main_table_chunk(
29
29
  Load a chunk of data from main table into memory, with expanded
30
30
  dims: (time, baseline, freq, pols)
31
31
 
32
- :param infile: Input MS path
33
- :param ddi: DDI to load chunk from
34
- :param chunk: specification of chunk to load
35
- :param ignore_msv2_cols: cols that should not be loaded (deprecated MSv2 or similar)
36
-
37
- :return: Xarray datasets with chunk of visibility data, one per DDI (spw_id, pol_setup_id) pair
32
+ Parameters
33
+ ----------
34
+ infile : str
35
+ Input MS path
36
+ ddi : int
37
+ DDI to load chunk from
38
+ chunk : Dict[str, slice]
39
+ specification of chunk to load
40
+ ignore_msv2_cols : Union[list, None] (Default value = None)
41
+ cols that should not be loaded (deprecated MSv2 or similar)
42
+
43
+ Returns
44
+ -------
45
+ xr.Dataset
46
+ Xarray datasets with chunk of visibility data, one per DDI (spw_id, pol_setup_id) pair
38
47
  """
39
48
 
40
49
  taql_where = f"where DATA_DESC_ID = {ddi}"
@@ -64,16 +73,24 @@ def load_expanded_ddi_chunk(
64
73
  xr.Dataset from a DII once the table and initial query(ies) have
65
74
  been opened successfully.
66
75
 
67
- :param infile: Input MS path
68
- :param tb_tool: table query contrained to one DDI and chunk time
69
- range
70
- :param taql_pre: TaQL query used for tb_tool, with some
71
- pre-selection of rows and columns
72
- :param chunk: specification of data chunk to load
73
- :param ignore_msv2_cols: propagated from calling funtions
74
-
75
- :return: An Xarray datasets with data variables as plain numpy
76
- arrays loaded directly from the MS columns
76
+ Parameters
77
+ ----------
78
+ infile : str
79
+ Input MS path
80
+ tb_tool : tables.table
81
+ table query contrained to one DDI and chunk time range
82
+ taql_pre : str
83
+ TaQL query used for tb_tool, with some pre-selection of rows and columns
84
+ chunk : Dict[str, slice]
85
+ specification of data chunk to load
86
+ ignore_msv2_cols : Union[list, None] (Default value = None)
87
+ propagated from calling funtions
88
+
89
+ Returns
90
+ -------
91
+ xr.Dataset
92
+ An Xarray dataset with data variables as plain numpy
93
+ arrays loaded directly from the MS columns
77
94
  """
78
95
 
79
96
  # read the specified chunk of data, figure out indices and lens
@@ -124,16 +141,30 @@ def load_ddi_cols_chunk(
124
141
  """
125
142
  For a given chunk and DDI, load all the MSv2 columns
126
143
 
127
- :param ctlen: length of the time axis/dim of the chunk
128
- :param cblen: length of the baseline axis of the chunk
129
- :param tidxs: time axis indices
130
- :param bidxs: baseline axis indices
131
- :param didxs: (effective) data indices, excluding missing baselines
132
- :param tb_tool: a table/TaQL query open and being used to load columns
133
- :param chunk: data chunk specification
134
- :param ignore_msv2_cols: propagated from calling funtions
144
+ Parameters
145
+ ----------
146
+ ctlen : int
147
+ length of the time axis/dim of the chunk
148
+ cblen : int
149
+ length of the baseline axis of the chunk
150
+ tidxs : np.ndarray
151
+ time axis indices
152
+ bidxs : np.ndarray
153
+ baseline axis indices
154
+ didxs : np.ndarray
155
+ (effective) data indices, excluding missing baselines
156
+ tb_tool : tables.table
157
+ a table/TaQL query open and being used to load columns
158
+ chunk : Dict[str, slice]
159
+ data chunk specification
160
+ ignore_msv2_cols : Union[list, None] (Default value = None)
161
+ propagated from calling funtions
162
+
163
+ Returns
164
+ -------
165
+ Dict[str, np.ndarray]
166
+ columns loaded into memory as np arrays
135
167
 
136
- :return: columns loaded into memory as np arrays
137
168
  """
138
169
  cols = tb_tool.colnames()
139
170
 
@@ -194,16 +225,23 @@ def load_ddi_cols_chunk(
194
225
 
195
226
  def get_chunk_times(
196
227
  taql_pre: str, chunk: Dict[str, slice]
197
- ) -> Tuple[np.ndarray, Tuple[int, int], int]:
228
+ ) -> Tuple[np.ndarray, Tuple[int, int]]:
198
229
  """
199
230
  Produces time col/axis related values for a chunk: unique times,
200
231
  start/stop times.
201
232
 
202
- :param chunk: specification of data chunk to load
203
- :param taql_pre: TaQL query used for tb_tool, with some pre-selection
204
- of rows and columns.
205
-
206
- :return: array of unique times + (firsr, last) time in the chunk
233
+ Parameters
234
+ ----------
235
+ taql_pre : str
236
+ TaQL query used for tb_tool, with some pre-selection
237
+ of rows and columns.
238
+ chunk : Dict[str, slice]
239
+ specification of data chunk to load
240
+
241
+ Returns
242
+ -------
243
+ Tuple[np.ndarray, Tuple[int, int]]
244
+ array of unique times + (firsr, last) time in the chunk
207
245
  """
208
246
 
209
247
  taql_utimes = f"select DISTINCT TIME from $mtable {taql_pre}"
@@ -246,15 +284,24 @@ def get_chunk_times(
246
284
  return utimes, times
247
285
 
248
286
 
249
- def get_chunk_baselines(tb_tool, chunk) -> Tuple[np.ndarray, Tuple[int, int], int]:
287
+ def get_chunk_baselines(
288
+ tb_tool: tables.table, chunk: Dict[str, slice]
289
+ ) -> Tuple[np.ndarray, Tuple[int, int]]:
250
290
  """
251
- Produces the basline col/axis related values for a chunk: an array
252
- of baselines and the start/stop baseline indices.
253
-
254
- :param tb_tool: table/query opened with prev selections (time)
255
- :param chunk: specification of data chunk to load
256
-
257
- :return: array of baselines + (first, last) baseline in the chunk
291
+ Produces the basline col/axis related values for a chunk: an array of
292
+ baselines and the start/stop baseline indices.
293
+
294
+ Parameters
295
+ ----------
296
+ tb_tool : tables.table
297
+ table/query opened with prev selections (time)
298
+ chunk : Dict[str, slice]
299
+ specification of data chunk to load
300
+
301
+ Returns
302
+ -------
303
+ Tuple[np.ndarray, Tuple[int, int]]
304
+ array of baselines + (first, last) baseline in the chunk
258
305
  """
259
306
  baselines = get_baselines(tb_tool)
260
307
 
@@ -274,22 +321,35 @@ def get_chunk_data_indices(
274
321
  times: Tuple[int, int],
275
322
  baselines: np.ndarray,
276
323
  blines: Tuple[int, int],
277
- ) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
324
+ ) -> Tuple[np.ndarray, np.ndarray, np.ndarray, str]:
278
325
  """
279
- Produces indices to pass to the casacore getcol(slice) functions
280
- to load the chunk of data. tidxs (time), bidxs (baseline),
281
- didxs (effective data indices, considering present/absent
282
- baselines).
326
+ Produces indices to pass to the casacore getcol(slice) functions to load
327
+ the chunk of data. tidxs (time), bidxs (baseline), didxs (effective data
328
+ indices, considering present/absent baselines).
283
329
 
284
330
  Time selection is added on top of that.
285
331
 
286
- :param utimes: array of times in the chunk
287
- :param times: start, stop time indices
288
- :param baselines: array of baselines inthe chunk
289
- :param blines: start, stop baseline indices
332
+ Parameters
333
+ ----------
334
+ taql_pre : str
335
+ TaQL query constraints to prepend/inject
336
+ chunk : Dict[str, slice]
337
+ specification of data chunk
338
+ utimes : np.ndarray
339
+ array of times in the chunk
340
+ times : Tuple[int, int]
341
+ start, stop time indices
342
+ baselines : np.ndarray
343
+ array of baselines inthe chunk
344
+ blines : Tuple[int, int]
345
+ start, stop baseline indices
346
+
347
+ Returns
348
+ -------
349
+ Tuple[np.ndarray, np.ndarray, np.ndarray]
350
+ indices along the time, baseline and data (time/baseline)
351
+ axes + the full where... string defined for this chunk
290
352
 
291
- :return: indices along the time, baseline and data (time/baseline)
292
- axes + the full where... defined for this chunk
293
353
  """
294
354
 
295
355
  taql_time = f"TIME BETWEEN {utimes[times[0]]} AND {utimes[times[1]]}"
@@ -333,14 +393,25 @@ def get_col_1d_pols(
333
393
  indices for the last dimension (either pol or freq).
334
394
  It also produces the adequate dimension names.
335
395
 
336
- :param cell_shape: shape of the column
337
- :param dims: full list of dataset dimensions
338
- :param chan_cnt: number of channels
339
- :param pol_cnt: number of pols
340
- :param chunk: data chunk specification
396
+ Parameters
397
+ ----------
398
+ cell_shape : Tuple[int]
399
+ shape of the column
400
+ dims : List[str]
401
+ full list of dataset dimensions
402
+ chan_cnt : int
403
+ number of channels
404
+ pol_cnt : int
405
+ number of pols
406
+ chunk : Dict[str, slice]
407
+ data chunk specification
408
+
409
+ Returns
410
+ -------
411
+ Tuple[Tuple[int, int], List[str]]
412
+ first and last pol/freq index of the chunk, and its
413
+ dimension names
341
414
 
342
- :return: first and last pol/freq index of the chunk, and its
343
- dimension name
344
415
  """
345
416
  if cell_shape == chan_cnt:
346
417
  # chan/freq
@@ -379,13 +450,23 @@ def get_col_2d_chans_pols(
379
450
  The dimension names can be assumed to be the full list of dims in
380
451
  visibilities (time, baseline, freq, pol).
381
452
 
382
- :param cell_shape: shape of the column
383
- :param chan_cnt: number of channels
384
- :param pol_cnt: number of pols
385
- :param chunk: data chunk specification
453
+ Parameters
454
+ ----------
455
+ cell_shape : Tuple[int]
456
+ shape of the column
457
+ chan_cnt : int
458
+ number of channels
459
+ pol_cnt : int
460
+ number of pols
461
+ chunk : Dict[str, slice]
462
+ data chunk specification
463
+
464
+ Returns
465
+ -------
466
+ Tuple[Tuple[int, int], Tuple[int, int]]
467
+ first and last index for freq (channel) and pol axes of
468
+ the chunk
386
469
 
387
- :return: first and last index for freq (channel) and pol axes of
388
- the chunk
389
470
  """
390
471
  if "freq" in chunk:
391
472
  chans = (