xradio 0.0.28__py3-none-any.whl → 0.0.30__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- xradio/__init__.py +5 -4
- xradio/_utils/array.py +90 -0
- xradio/_utils/zarr/common.py +48 -3
- xradio/image/_util/zarr.py +4 -1
- xradio/schema/__init__.py +24 -6
- xradio/schema/bases.py +440 -2
- xradio/schema/check.py +96 -55
- xradio/schema/dataclass.py +123 -27
- xradio/schema/metamodel.py +21 -4
- xradio/schema/typing.py +33 -18
- xradio/vis/__init__.py +5 -2
- xradio/vis/_processing_set.py +71 -32
- xradio/vis/_vis_utils/_ms/_tables/create_field_and_source_xds.py +710 -0
- xradio/vis/_vis_utils/_ms/_tables/load.py +23 -10
- xradio/vis/_vis_utils/_ms/_tables/load_main_table.py +145 -64
- xradio/vis/_vis_utils/_ms/_tables/read.py +747 -172
- xradio/vis/_vis_utils/_ms/_tables/read_main_table.py +173 -44
- xradio/vis/_vis_utils/_ms/_tables/read_subtables.py +79 -28
- xradio/vis/_vis_utils/_ms/_tables/write.py +102 -45
- xradio/vis/_vis_utils/_ms/_tables/write_exp_api.py +127 -65
- xradio/vis/_vis_utils/_ms/chunks.py +58 -21
- xradio/vis/_vis_utils/_ms/conversion.py +582 -102
- xradio/vis/_vis_utils/_ms/descr.py +52 -20
- xradio/vis/_vis_utils/_ms/msv2_to_msv4_meta.py +72 -35
- xradio/vis/_vis_utils/_ms/msv4_infos.py +0 -59
- xradio/vis/_vis_utils/_ms/msv4_sub_xdss.py +76 -9
- xradio/vis/_vis_utils/_ms/optimised_functions.py +0 -46
- xradio/vis/_vis_utils/_ms/partition_queries.py +308 -119
- xradio/vis/_vis_utils/_ms/partitions.py +82 -25
- xradio/vis/_vis_utils/_ms/subtables.py +32 -14
- xradio/vis/_vis_utils/_utils/partition_attrs.py +30 -11
- xradio/vis/_vis_utils/_utils/xds_helper.py +136 -45
- xradio/vis/_vis_utils/_zarr/read.py +60 -22
- xradio/vis/_vis_utils/_zarr/write.py +83 -9
- xradio/vis/_vis_utils/ms.py +48 -29
- xradio/vis/_vis_utils/zarr.py +44 -20
- xradio/vis/convert_msv2_to_processing_set.py +43 -32
- xradio/vis/load_processing_set.py +38 -61
- xradio/vis/read_processing_set.py +64 -96
- xradio/vis/schema.py +687 -0
- xradio/vis/vis_io.py +75 -43
- {xradio-0.0.28.dist-info → xradio-0.0.30.dist-info}/LICENSE.txt +6 -1
- {xradio-0.0.28.dist-info → xradio-0.0.30.dist-info}/METADATA +10 -5
- xradio-0.0.30.dist-info/RECORD +73 -0
- {xradio-0.0.28.dist-info → xradio-0.0.30.dist-info}/WHEEL +1 -1
- xradio/vis/model.py +0 -497
- xradio-0.0.28.dist-info/RECORD +0 -71
- {xradio-0.0.28.dist-info → xradio-0.0.30.dist-info}/top_level.txt +0 -0
|
@@ -17,16 +17,29 @@ def load_col_chunk(
|
|
|
17
17
|
"""
|
|
18
18
|
Loads a slice of a col (using casacore getcol(slice))
|
|
19
19
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
20
|
+
Parameters
|
|
21
|
+
----------
|
|
22
|
+
tb_tool : tables.table
|
|
23
|
+
a table/TaQL query open and being used to load columns
|
|
24
|
+
col : str
|
|
25
|
+
colum to load
|
|
26
|
+
cshape : Tuple[int]
|
|
27
|
+
shape of the resulting col data chunk
|
|
28
|
+
tidxs : np.ndarray
|
|
29
|
+
time axis indices
|
|
30
|
+
bidxs : np.ndarray
|
|
31
|
+
baseline axis indices
|
|
32
|
+
didxs : np.ndarray
|
|
33
|
+
effective) data indices, excluding missing baselines
|
|
34
|
+
d1 : Tuple[int, int]
|
|
35
|
+
indices to load on dimension 1 (None=all, pols or chans)
|
|
36
|
+
d2 : Tuple[int, int]
|
|
37
|
+
indices to load on dimension 2 (None=all, pols)
|
|
38
|
+
|
|
39
|
+
Returns
|
|
40
|
+
-------
|
|
41
|
+
np.ndarray
|
|
42
|
+
data array loaded directly with casacore getcol/getcolslice
|
|
30
43
|
"""
|
|
31
44
|
|
|
32
45
|
if (len(cshape) == 2) or (col == "UVW"):
|
|
@@ -16,7 +16,7 @@ from xradio.vis._vis_utils._ms._tables.read_main_table import (
|
|
|
16
16
|
get_baselines,
|
|
17
17
|
get_baseline_indices,
|
|
18
18
|
)
|
|
19
|
-
from xradio.
|
|
19
|
+
from xradio._utils.array import unique_1d
|
|
20
20
|
|
|
21
21
|
|
|
22
22
|
def load_expanded_main_table_chunk(
|
|
@@ -29,12 +29,21 @@ def load_expanded_main_table_chunk(
|
|
|
29
29
|
Load a chunk of data from main table into memory, with expanded
|
|
30
30
|
dims: (time, baseline, freq, pols)
|
|
31
31
|
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
32
|
+
Parameters
|
|
33
|
+
----------
|
|
34
|
+
infile : str
|
|
35
|
+
Input MS path
|
|
36
|
+
ddi : int
|
|
37
|
+
DDI to load chunk from
|
|
38
|
+
chunk : Dict[str, slice]
|
|
39
|
+
specification of chunk to load
|
|
40
|
+
ignore_msv2_cols : Union[list, None] (Default value = None)
|
|
41
|
+
cols that should not be loaded (deprecated MSv2 or similar)
|
|
42
|
+
|
|
43
|
+
Returns
|
|
44
|
+
-------
|
|
45
|
+
xr.Dataset
|
|
46
|
+
Xarray datasets with chunk of visibility data, one per DDI (spw_id, pol_setup_id) pair
|
|
38
47
|
"""
|
|
39
48
|
|
|
40
49
|
taql_where = f"where DATA_DESC_ID = {ddi}"
|
|
@@ -64,16 +73,24 @@ def load_expanded_ddi_chunk(
|
|
|
64
73
|
xr.Dataset from a DII once the table and initial query(ies) have
|
|
65
74
|
been opened successfully.
|
|
66
75
|
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
:
|
|
76
|
-
|
|
76
|
+
Parameters
|
|
77
|
+
----------
|
|
78
|
+
infile : str
|
|
79
|
+
Input MS path
|
|
80
|
+
tb_tool : tables.table
|
|
81
|
+
table query contrained to one DDI and chunk time range
|
|
82
|
+
taql_pre : str
|
|
83
|
+
TaQL query used for tb_tool, with some pre-selection of rows and columns
|
|
84
|
+
chunk : Dict[str, slice]
|
|
85
|
+
specification of data chunk to load
|
|
86
|
+
ignore_msv2_cols : Union[list, None] (Default value = None)
|
|
87
|
+
propagated from calling funtions
|
|
88
|
+
|
|
89
|
+
Returns
|
|
90
|
+
-------
|
|
91
|
+
xr.Dataset
|
|
92
|
+
An Xarray dataset with data variables as plain numpy
|
|
93
|
+
arrays loaded directly from the MS columns
|
|
77
94
|
"""
|
|
78
95
|
|
|
79
96
|
# read the specified chunk of data, figure out indices and lens
|
|
@@ -124,16 +141,30 @@ def load_ddi_cols_chunk(
|
|
|
124
141
|
"""
|
|
125
142
|
For a given chunk and DDI, load all the MSv2 columns
|
|
126
143
|
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
144
|
+
Parameters
|
|
145
|
+
----------
|
|
146
|
+
ctlen : int
|
|
147
|
+
length of the time axis/dim of the chunk
|
|
148
|
+
cblen : int
|
|
149
|
+
length of the baseline axis of the chunk
|
|
150
|
+
tidxs : np.ndarray
|
|
151
|
+
time axis indices
|
|
152
|
+
bidxs : np.ndarray
|
|
153
|
+
baseline axis indices
|
|
154
|
+
didxs : np.ndarray
|
|
155
|
+
(effective) data indices, excluding missing baselines
|
|
156
|
+
tb_tool : tables.table
|
|
157
|
+
a table/TaQL query open and being used to load columns
|
|
158
|
+
chunk : Dict[str, slice]
|
|
159
|
+
data chunk specification
|
|
160
|
+
ignore_msv2_cols : Union[list, None] (Default value = None)
|
|
161
|
+
propagated from calling funtions
|
|
162
|
+
|
|
163
|
+
Returns
|
|
164
|
+
-------
|
|
165
|
+
Dict[str, np.ndarray]
|
|
166
|
+
columns loaded into memory as np arrays
|
|
135
167
|
|
|
136
|
-
:return: columns loaded into memory as np arrays
|
|
137
168
|
"""
|
|
138
169
|
cols = tb_tool.colnames()
|
|
139
170
|
|
|
@@ -194,16 +225,23 @@ def load_ddi_cols_chunk(
|
|
|
194
225
|
|
|
195
226
|
def get_chunk_times(
|
|
196
227
|
taql_pre: str, chunk: Dict[str, slice]
|
|
197
|
-
) -> Tuple[np.ndarray, Tuple[int, int]
|
|
228
|
+
) -> Tuple[np.ndarray, Tuple[int, int]]:
|
|
198
229
|
"""
|
|
199
230
|
Produces time col/axis related values for a chunk: unique times,
|
|
200
231
|
start/stop times.
|
|
201
232
|
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
233
|
+
Parameters
|
|
234
|
+
----------
|
|
235
|
+
taql_pre : str
|
|
236
|
+
TaQL query used for tb_tool, with some pre-selection
|
|
237
|
+
of rows and columns.
|
|
238
|
+
chunk : Dict[str, slice]
|
|
239
|
+
specification of data chunk to load
|
|
240
|
+
|
|
241
|
+
Returns
|
|
242
|
+
-------
|
|
243
|
+
Tuple[np.ndarray, Tuple[int, int]]
|
|
244
|
+
array of unique times + (firsr, last) time in the chunk
|
|
207
245
|
"""
|
|
208
246
|
|
|
209
247
|
taql_utimes = f"select DISTINCT TIME from $mtable {taql_pre}"
|
|
@@ -246,15 +284,24 @@ def get_chunk_times(
|
|
|
246
284
|
return utimes, times
|
|
247
285
|
|
|
248
286
|
|
|
249
|
-
def get_chunk_baselines(
|
|
287
|
+
def get_chunk_baselines(
|
|
288
|
+
tb_tool: tables.table, chunk: Dict[str, slice]
|
|
289
|
+
) -> Tuple[np.ndarray, Tuple[int, int]]:
|
|
250
290
|
"""
|
|
251
|
-
Produces the basline col/axis related values for a chunk: an array
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
291
|
+
Produces the basline col/axis related values for a chunk: an array of
|
|
292
|
+
baselines and the start/stop baseline indices.
|
|
293
|
+
|
|
294
|
+
Parameters
|
|
295
|
+
----------
|
|
296
|
+
tb_tool : tables.table
|
|
297
|
+
table/query opened with prev selections (time)
|
|
298
|
+
chunk : Dict[str, slice]
|
|
299
|
+
specification of data chunk to load
|
|
300
|
+
|
|
301
|
+
Returns
|
|
302
|
+
-------
|
|
303
|
+
Tuple[np.ndarray, Tuple[int, int]]
|
|
304
|
+
array of baselines + (first, last) baseline in the chunk
|
|
258
305
|
"""
|
|
259
306
|
baselines = get_baselines(tb_tool)
|
|
260
307
|
|
|
@@ -274,22 +321,35 @@ def get_chunk_data_indices(
|
|
|
274
321
|
times: Tuple[int, int],
|
|
275
322
|
baselines: np.ndarray,
|
|
276
323
|
blines: Tuple[int, int],
|
|
277
|
-
) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
|
|
324
|
+
) -> Tuple[np.ndarray, np.ndarray, np.ndarray, str]:
|
|
278
325
|
"""
|
|
279
|
-
Produces indices to pass to the casacore getcol(slice) functions
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
baselines).
|
|
326
|
+
Produces indices to pass to the casacore getcol(slice) functions to load
|
|
327
|
+
the chunk of data. tidxs (time), bidxs (baseline), didxs (effective data
|
|
328
|
+
indices, considering present/absent baselines).
|
|
283
329
|
|
|
284
330
|
Time selection is added on top of that.
|
|
285
331
|
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
332
|
+
Parameters
|
|
333
|
+
----------
|
|
334
|
+
taql_pre : str
|
|
335
|
+
TaQL query constraints to prepend/inject
|
|
336
|
+
chunk : Dict[str, slice]
|
|
337
|
+
specification of data chunk
|
|
338
|
+
utimes : np.ndarray
|
|
339
|
+
array of times in the chunk
|
|
340
|
+
times : Tuple[int, int]
|
|
341
|
+
start, stop time indices
|
|
342
|
+
baselines : np.ndarray
|
|
343
|
+
array of baselines inthe chunk
|
|
344
|
+
blines : Tuple[int, int]
|
|
345
|
+
start, stop baseline indices
|
|
346
|
+
|
|
347
|
+
Returns
|
|
348
|
+
-------
|
|
349
|
+
Tuple[np.ndarray, np.ndarray, np.ndarray]
|
|
350
|
+
indices along the time, baseline and data (time/baseline)
|
|
351
|
+
axes + the full where... string defined for this chunk
|
|
290
352
|
|
|
291
|
-
:return: indices along the time, baseline and data (time/baseline)
|
|
292
|
-
axes + the full where... defined for this chunk
|
|
293
353
|
"""
|
|
294
354
|
|
|
295
355
|
taql_time = f"TIME BETWEEN {utimes[times[0]]} AND {utimes[times[1]]}"
|
|
@@ -333,14 +393,25 @@ def get_col_1d_pols(
|
|
|
333
393
|
indices for the last dimension (either pol or freq).
|
|
334
394
|
It also produces the adequate dimension names.
|
|
335
395
|
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
396
|
+
Parameters
|
|
397
|
+
----------
|
|
398
|
+
cell_shape : Tuple[int]
|
|
399
|
+
shape of the column
|
|
400
|
+
dims : List[str]
|
|
401
|
+
full list of dataset dimensions
|
|
402
|
+
chan_cnt : int
|
|
403
|
+
number of channels
|
|
404
|
+
pol_cnt : int
|
|
405
|
+
number of pols
|
|
406
|
+
chunk : Dict[str, slice]
|
|
407
|
+
data chunk specification
|
|
408
|
+
|
|
409
|
+
Returns
|
|
410
|
+
-------
|
|
411
|
+
Tuple[Tuple[int, int], List[str]]
|
|
412
|
+
first and last pol/freq index of the chunk, and its
|
|
413
|
+
dimension names
|
|
341
414
|
|
|
342
|
-
:return: first and last pol/freq index of the chunk, and its
|
|
343
|
-
dimension name
|
|
344
415
|
"""
|
|
345
416
|
if cell_shape == chan_cnt:
|
|
346
417
|
# chan/freq
|
|
@@ -379,13 +450,23 @@ def get_col_2d_chans_pols(
|
|
|
379
450
|
The dimension names can be assumed to be the full list of dims in
|
|
380
451
|
visibilities (time, baseline, freq, pol).
|
|
381
452
|
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
453
|
+
Parameters
|
|
454
|
+
----------
|
|
455
|
+
cell_shape : Tuple[int]
|
|
456
|
+
shape of the column
|
|
457
|
+
chan_cnt : int
|
|
458
|
+
number of channels
|
|
459
|
+
pol_cnt : int
|
|
460
|
+
number of pols
|
|
461
|
+
chunk : Dict[str, slice]
|
|
462
|
+
data chunk specification
|
|
463
|
+
|
|
464
|
+
Returns
|
|
465
|
+
-------
|
|
466
|
+
Tuple[Tuple[int, int], Tuple[int, int]]
|
|
467
|
+
first and last index for freq (channel) and pol axes of
|
|
468
|
+
the chunk
|
|
386
469
|
|
|
387
|
-
:return: first and last index for freq (channel) and pol axes of
|
|
388
|
-
the chunk
|
|
389
470
|
"""
|
|
390
471
|
if "freq" in chunk:
|
|
391
472
|
chans = (
|