xradio 0.0.38__py3-none-any.whl → 0.0.40__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- xradio/__init__.py +1 -1
- xradio/_utils/common.py +41 -0
- xradio/_utils/schema.py +14 -3
- xradio/image/_util/_casacore/xds_from_casacore.py +1 -1
- xradio/image/_util/_zarr/xds_from_zarr.py +1 -1
- xradio/image/_util/_zarr/zarr_low_level.py +1 -1
- xradio/vis/_vis_utils/_ms/_tables/load.py +5 -6
- xradio/vis/_vis_utils/_ms/_tables/load_main_table.py +1 -1
- xradio/vis/_vis_utils/_ms/_tables/read.py +58 -52
- xradio/vis/_vis_utils/_ms/_tables/read_main_table.py +1 -1
- xradio/vis/_vis_utils/_ms/_tables/read_subtables.py +1 -1
- xradio/vis/_vis_utils/_ms/_tables/write.py +1 -1
- xradio/vis/_vis_utils/_ms/conversion.py +1 -1
- xradio/vis/_vis_utils/_ms/create_antenna_xds.py +1 -1
- xradio/vis/_vis_utils/_ms/create_field_and_source_xds.py +208 -175
- xradio/vis/_vis_utils/_ms/msv2_to_msv4_meta.py +1 -1
- xradio/vis/_vis_utils/_ms/msv4_sub_xdss.py +133 -178
- xradio/vis/_vis_utils/_ms/partition_queries.py +1 -1
- xradio/vis/_vis_utils/_ms/partitions.py +1 -1
- xradio/vis/_vis_utils/_ms/subtables.py +1 -1
- xradio/vis/_vis_utils/_utils/xds_helper.py +10 -12
- xradio/vis/_vis_utils/_zarr/read.py +1 -1
- xradio/vis/_vis_utils/_zarr/write.py +1 -1
- xradio/vis/_vis_utils/ms.py +1 -1
- xradio/vis/_vis_utils/zarr.py +1 -1
- xradio/vis/convert_msv2_to_processing_set.py +1 -1
- xradio/vis/read_processing_set.py +1 -1
- {xradio-0.0.38.dist-info → xradio-0.0.40.dist-info}/METADATA +3 -2
- {xradio-0.0.38.dist-info → xradio-0.0.40.dist-info}/RECORD +32 -32
- {xradio-0.0.38.dist-info → xradio-0.0.40.dist-info}/WHEEL +1 -1
- {xradio-0.0.38.dist-info → xradio-0.0.40.dist-info}/LICENSE.txt +0 -0
- {xradio-0.0.38.dist-info → xradio-0.0.40.dist-info}/top_level.txt +0 -0
|
@@ -1,11 +1,14 @@
|
|
|
1
|
-
import
|
|
1
|
+
import toolviper.utils.logger as logger
|
|
2
2
|
import time
|
|
3
3
|
from typing import Tuple, Union
|
|
4
4
|
|
|
5
5
|
import numpy as np
|
|
6
6
|
import xarray as xr
|
|
7
7
|
|
|
8
|
-
from xradio._utils.schema import
|
|
8
|
+
from xradio._utils.schema import (
|
|
9
|
+
column_description_casacore_to_msv4_measure,
|
|
10
|
+
convert_generic_xds_to_xradio_schema,
|
|
11
|
+
)
|
|
9
12
|
from .subtables import subt_rename_ids
|
|
10
13
|
from ._tables.read import make_taql_where_between_min_max, load_generic_table
|
|
11
14
|
|
|
@@ -70,37 +73,22 @@ def create_weather_xds(in_file: str):
|
|
|
70
73
|
xr.Dataset
|
|
71
74
|
Weather Xarray Dataset.
|
|
72
75
|
"""
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
"
|
|
78
|
-
"
|
|
79
|
-
"
|
|
80
|
-
"
|
|
81
|
-
"
|
|
82
|
-
"
|
|
83
|
-
"
|
|
84
|
-
"WIND_SPEED": "WIND_SPEED",
|
|
85
|
-
}
|
|
86
|
-
data_variable_dims = {
|
|
87
|
-
"H2O": ["station_id", "time"],
|
|
88
|
-
"IONOS_ELECTRON": ["station_id", "time"],
|
|
89
|
-
"PRESSURE": ["station_id", "time"],
|
|
90
|
-
"REL_HUMIDITY": ["station_id", "time"],
|
|
91
|
-
"TEMPERATURE": ["station_id", "time"],
|
|
92
|
-
"DEW_POINT": ["station_id", "time"],
|
|
93
|
-
"WIND_DIRECTION": ["station_id", "time"],
|
|
94
|
-
"WIND_SPEED": ["station_id", "time"],
|
|
95
|
-
}
|
|
96
|
-
to_new_coord_names = {
|
|
97
|
-
# No MS data cols are turned into xds coords
|
|
98
|
-
}
|
|
99
|
-
coord_dims = {
|
|
100
|
-
# No MS data cols are turned into xds coords
|
|
76
|
+
|
|
77
|
+
dims_station_time = ["station_id", "time"]
|
|
78
|
+
to_new_data_variables = {
|
|
79
|
+
"H20": ["H2O", dims_station_time],
|
|
80
|
+
"IONOS_ELECTRON": ["IONOS_ELECTRON", dims_station_time],
|
|
81
|
+
"PRESSURE": ["PRESSURE", dims_station_time],
|
|
82
|
+
"REL_HUMIDITY": ["REL_HUMIDITY", dims_station_time],
|
|
83
|
+
"TEMPERATURE": ["TEMPERATURE", dims_station_time],
|
|
84
|
+
"DEW_POINT": ["DEW_POINT", dims_station_time],
|
|
85
|
+
"WIND_DIRECTION": ["WIND_DIRECTION", dims_station_time],
|
|
86
|
+
"WIND_SPEED": ["WIND_SPEED", dims_station_time],
|
|
101
87
|
}
|
|
102
|
-
|
|
103
|
-
|
|
88
|
+
|
|
89
|
+
to_new_coords = {
|
|
90
|
+
"ANTENNA_ID": ["station_id", ["station_id"]],
|
|
91
|
+
"TIME": ["time", ["time"]],
|
|
104
92
|
}
|
|
105
93
|
|
|
106
94
|
# Read WEATHER table into a Xarray Dataset.
|
|
@@ -113,74 +101,88 @@ def create_weather_xds(in_file: str):
|
|
|
113
101
|
except ValueError as _exc:
|
|
114
102
|
return None
|
|
115
103
|
|
|
116
|
-
generic_weather_xds = generic_weather_xds.rename_dims(to_new_dim_names)
|
|
117
|
-
|
|
118
|
-
weather_column_description = generic_weather_xds.attrs["other"]["msv2"][
|
|
119
|
-
"ctds_attrs"
|
|
120
|
-
]["column_descriptions"]
|
|
121
|
-
# ['ANTENNA_ID', 'TIME', 'INTERVAL', 'H2O', 'IONOS_ELECTRON',
|
|
122
|
-
# 'PRESSURE', 'REL_HUMIDITY', 'TEMPERATURE', 'DEW_POINT',
|
|
123
|
-
# 'WIND_DIRECTION', 'WIND_SPEED']
|
|
124
104
|
weather_xds = xr.Dataset(attrs={"type": "weather"})
|
|
125
|
-
|
|
126
|
-
|
|
105
|
+
weather_xds = convert_generic_xds_to_xradio_schema(
|
|
106
|
+
generic_weather_xds, weather_xds, to_new_data_variables, to_new_coords
|
|
127
107
|
)
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
for key in generic_weather_xds:
|
|
133
|
-
msv4_measure = column_description_casacore_to_msv4_measure(
|
|
134
|
-
weather_column_description[key.upper()]
|
|
135
|
-
)
|
|
136
|
-
if key in to_new_data_variable_names:
|
|
137
|
-
var_name = to_new_data_variable_names[key]
|
|
138
|
-
weather_xds[var_name] = xr.DataArray(
|
|
139
|
-
generic_weather_xds[key].data, dims=data_variable_dims[key]
|
|
140
|
-
)
|
|
141
|
-
|
|
142
|
-
if msv4_measure:
|
|
143
|
-
weather_xds[var_name].attrs.update(msv4_measure)
|
|
144
|
-
|
|
145
|
-
if key in ["INTERVAL"]:
|
|
146
|
-
weather_xds[var_name].attrs.update({"units": ["s"], "type": "quantity"})
|
|
147
|
-
elif key in ["H2O"]:
|
|
148
|
-
weather_xds[var_name].attrs.update(
|
|
149
|
-
{"units": ["/m^2"], "type": "quantity"}
|
|
150
|
-
)
|
|
151
|
-
elif key in ["IONOS_ELECTRON"]:
|
|
152
|
-
weather_xds[var_name].attrs.update(
|
|
153
|
-
{"units": ["/m^2"], "type": "quantity"}
|
|
154
|
-
)
|
|
155
|
-
elif key in ["PRESSURE"]:
|
|
156
|
-
weather_xds[var_name].attrs.update(
|
|
157
|
-
{"units": ["Pa"], "type": "quantity"}
|
|
158
|
-
)
|
|
159
|
-
elif key in ["REL_HUMIDITY"]:
|
|
160
|
-
weather_xds[var_name].attrs.update({"units": ["%"], "type": "quantity"})
|
|
161
|
-
elif key in ["TEMPERATURE"]:
|
|
162
|
-
weather_xds[var_name].attrs.update({"units": ["K"], "type": "quantity"})
|
|
163
|
-
elif key in ["DEW_POINT"]:
|
|
164
|
-
weather_xds[var_name].attrs.update({"units": ["K"], "type": "quantity"})
|
|
165
|
-
elif key in ["WIND_DIRECTION"]:
|
|
166
|
-
weather_xds[var_name].attrs.update(
|
|
167
|
-
{"units": ["rad"], "type": "quantity"}
|
|
168
|
-
)
|
|
169
|
-
elif key in ["WIND_SPEED"]:
|
|
170
|
-
weather_xds[var_name].attrs.update(
|
|
171
|
-
{"units": ["m/s"], "type": "quantity"}
|
|
172
|
-
)
|
|
173
|
-
|
|
174
|
-
if key in to_new_coord_names:
|
|
175
|
-
coords[to_new_coord_names[key]] = (
|
|
176
|
-
coord_dims[key],
|
|
177
|
-
generic_weather_xds[key].data,
|
|
178
|
-
)
|
|
179
|
-
|
|
180
|
-
weather_xds = weather_xds.assign_coords(coords)
|
|
108
|
+
|
|
109
|
+
# correct expected types
|
|
110
|
+
weather_xds["station_id"] = weather_xds["station_id"].astype(np.int64)
|
|
111
|
+
|
|
181
112
|
return weather_xds
|
|
182
113
|
|
|
183
114
|
|
|
115
|
+
def correct_generic_pointing_xds(
|
|
116
|
+
generic_pointing_xds: xr.Dataset, to_new_data_variables: dict[str, list]
|
|
117
|
+
) -> xr.Dataset:
|
|
118
|
+
"""
|
|
119
|
+
Takes a (generic) pointing_xds as read from a POINTING subtable of an MSv2
|
|
120
|
+
and tries to correct several deviations from the MSv2 specs seen in
|
|
121
|
+
common test data.
|
|
122
|
+
The problems fixed here include wrong dimensions:
|
|
123
|
+
- for example transposed dimensions with respect to the MSv2 specs (output
|
|
124
|
+
from CASA simulator),
|
|
125
|
+
- missing/additional unexpected dimensions when some of the columns are
|
|
126
|
+
empty (in the sense of "empty casacore cells").
|
|
127
|
+
|
|
128
|
+
This function modifies the data arrays of the data vars affected by such
|
|
129
|
+
issues.
|
|
130
|
+
|
|
131
|
+
Parameters
|
|
132
|
+
----------
|
|
133
|
+
generic_pointing_xds: xr.Dataset
|
|
134
|
+
The generic pointing dataset (loaded from MSv2) to be fixed
|
|
135
|
+
to_new_data_variables: dict
|
|
136
|
+
The dict used for convert_generic_xds_to_xradio_schema, which gives all
|
|
137
|
+
the data variables relevant for the final MSv4 dataset.
|
|
138
|
+
|
|
139
|
+
Returns:
|
|
140
|
+
--------
|
|
141
|
+
xr.Dataset
|
|
142
|
+
Corrected dataset with dimensions conforming to MSv2 specs.
|
|
143
|
+
"""
|
|
144
|
+
|
|
145
|
+
correct_pointing_xds = generic_pointing_xds.copy()
|
|
146
|
+
|
|
147
|
+
for key in generic_pointing_xds:
|
|
148
|
+
if key in to_new_data_variables:
|
|
149
|
+
data_var_name = to_new_data_variables[key]
|
|
150
|
+
# Corrects dim sizes of "empty cell" variables, such as empty DIRECTION, TARGET, etc.
|
|
151
|
+
if (
|
|
152
|
+
"dim_2" in generic_pointing_xds.sizes
|
|
153
|
+
and generic_pointing_xds.sizes["dim_2"] == 0
|
|
154
|
+
):
|
|
155
|
+
# When all direction variables are "empty"
|
|
156
|
+
data_var_data = xr.DataArray(
|
|
157
|
+
[[[[np.nan, np.nan]]]],
|
|
158
|
+
dims=generic_pointing_xds.dims,
|
|
159
|
+
).isel(n_polynomial=0, drop=True)
|
|
160
|
+
correct_pointing_xds[data_var_name].data = data_var_data
|
|
161
|
+
|
|
162
|
+
elif (
|
|
163
|
+
"dir" in generic_pointing_xds.sizes
|
|
164
|
+
and generic_pointing_xds.sizes["dir"] == 0
|
|
165
|
+
):
|
|
166
|
+
# When some direction variables are "empty" but some are populated properly
|
|
167
|
+
if "dim_2" in generic_pointing_xds[key].sizes:
|
|
168
|
+
data_var_data = xr.DataArray(
|
|
169
|
+
generic_pointing_xds[key].values,
|
|
170
|
+
dims=generic_pointing_xds[key].dims,
|
|
171
|
+
)
|
|
172
|
+
else:
|
|
173
|
+
shape = tuple(
|
|
174
|
+
generic_pointing_xds.sizes[dim]
|
|
175
|
+
for dim in ["TIME", "ANTENNA_ID"]
|
|
176
|
+
) + (2,)
|
|
177
|
+
data_var_data = xr.DataArray(
|
|
178
|
+
np.full(shape, np.nan),
|
|
179
|
+
dims=generic_pointing_xds[key].dims,
|
|
180
|
+
)
|
|
181
|
+
correct_pointing_xds[data_var_name].data = data_var_data
|
|
182
|
+
|
|
183
|
+
return correct_pointing_xds
|
|
184
|
+
|
|
185
|
+
|
|
184
186
|
def create_pointing_xds(
|
|
185
187
|
in_file: str,
|
|
186
188
|
ant_xds_name_ids: xr.DataArray,
|
|
@@ -211,49 +213,43 @@ def create_pointing_xds(
|
|
|
211
213
|
"""
|
|
212
214
|
start = time.time()
|
|
213
215
|
|
|
214
|
-
# Dictionaries that define the conversion from MSv2 to MSv4:
|
|
215
|
-
to_new_data_variable_names = {
|
|
216
|
-
# "name": "NAME", # removed
|
|
217
|
-
# "time_origin": "TIME_ORIGIN", # removed?
|
|
218
|
-
"DIRECTION": "BEAM_POINTING",
|
|
219
|
-
"ENCODER": "DISH_MEASURED_POINTING",
|
|
220
|
-
"TARGET": "TARGET", # => attribute?
|
|
221
|
-
"POINTING_OFFSET": "POINTING_OFFSET",
|
|
222
|
-
"SOURCE_OFFSET": "SOURCE_OFFSET",
|
|
223
|
-
# "pointing_model_id": "POINTING_MODEL_ID", # removed
|
|
224
|
-
# "tracking": "TRACKING", # => attribute
|
|
225
|
-
# "on_source": "ON_SOURCE", # removed
|
|
226
|
-
"OVER_THE_TOP": "OVER_THE_TOP",
|
|
227
|
-
}
|
|
228
216
|
time_ant_dims = ["time", "antenna_name"]
|
|
229
217
|
time_ant_dir_dims = time_ant_dims + ["sky_dir_label"]
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
"
|
|
235
|
-
"
|
|
236
|
-
"
|
|
237
|
-
"
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
"
|
|
218
|
+
to_new_data_variables = {
|
|
219
|
+
"DIRECTION": ["BEAM_POINTING", time_ant_dir_dims],
|
|
220
|
+
"ENCODER": ["DISH_MEASURED_POINTING", time_ant_dir_dims],
|
|
221
|
+
# => attribute?
|
|
222
|
+
"TARGET": ["TARGET", time_ant_dir_dims],
|
|
223
|
+
"POINTING_OFFSET": ["POINTING_OFFSET", time_ant_dir_dims],
|
|
224
|
+
"SOURCE_OFFSET": ["SOURCE_OFFSET", time_ant_dir_dims],
|
|
225
|
+
"OVER_THE_TOP": ["OVER_THE_TOP", time_ant_dims],
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
to_new_coords = {
|
|
229
|
+
"TIME": ["time", ["time"]],
|
|
230
|
+
# "ANTENNA_ID": ["antenna_name", ["antenna_name"]],
|
|
231
|
+
"dim_2": ["sky_dir_label", ["sky_dir_label"]],
|
|
242
232
|
}
|
|
243
|
-
# Unused here
|
|
244
|
-
# to_new_coord_names = {"ra/dec": "direction"}
|
|
245
|
-
# coord_dims = {}
|
|
246
233
|
|
|
247
234
|
taql_time_range = make_taql_where_between_min_max(
|
|
248
235
|
time_min_max, in_file, "POINTING", "TIME"
|
|
249
236
|
)
|
|
237
|
+
|
|
238
|
+
if taql_time_range is None:
|
|
239
|
+
taql_where = f"WHERE (ANTENNA_ID IN [{','.join(map(str, ant_xds_name_ids.antenna_id.values))}])"
|
|
240
|
+
else:
|
|
241
|
+
taql_where = (
|
|
242
|
+
taql_time_range
|
|
243
|
+
+ f" AND (ANTENNA_ID IN [{','.join(map(str, ant_xds_name_ids.antenna_id.values))}])"
|
|
244
|
+
)
|
|
250
245
|
# Read POINTING table into a Xarray Dataset.
|
|
251
246
|
generic_pointing_xds = load_generic_table(
|
|
252
247
|
in_file,
|
|
253
248
|
"POINTING",
|
|
254
249
|
rename_ids=subt_rename_ids["POINTING"],
|
|
255
|
-
taql_where=
|
|
250
|
+
taql_where=taql_where,
|
|
256
251
|
)
|
|
252
|
+
|
|
257
253
|
if not generic_pointing_xds.data_vars:
|
|
258
254
|
# apparently empty MS/POINTING table => produce empty xds
|
|
259
255
|
return xr.Dataset()
|
|
@@ -264,69 +260,23 @@ def create_pointing_xds(
|
|
|
264
260
|
if size == 1:
|
|
265
261
|
generic_pointing_xds = generic_pointing_xds.sel({"n_polynomial": 0})
|
|
266
262
|
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
263
|
+
generic_pointing_xds = correct_generic_pointing_xds(
|
|
264
|
+
generic_pointing_xds, to_new_data_variables
|
|
265
|
+
)
|
|
270
266
|
|
|
271
267
|
pointing_xds = xr.Dataset(attrs={"type": "pointing"})
|
|
272
|
-
for key in generic_pointing_xds:
|
|
273
|
-
if key in to_new_data_variable_names:
|
|
274
|
-
data_var_name = to_new_data_variable_names[key]
|
|
275
|
-
# Corrects dim sizes of "empty cell" variables, such as empty DIRECTION, TARGET, etc.
|
|
276
|
-
# TODO: this should be moved to a function when/if stable - perhaps 'correct_generic_pointing_xds'
|
|
277
|
-
if (
|
|
278
|
-
"dim_2" in generic_pointing_xds.sizes
|
|
279
|
-
and generic_pointing_xds.sizes["dim_2"] == 0
|
|
280
|
-
):
|
|
281
|
-
# When all direction variables are "empty"
|
|
282
|
-
data_var_data = xr.DataArray(
|
|
283
|
-
[[[[np.nan, np.nan]]]],
|
|
284
|
-
dims=generic_pointing_xds.dims,
|
|
285
|
-
).isel(n_polynomial=0, drop=True)
|
|
286
|
-
elif (
|
|
287
|
-
"dir" in generic_pointing_xds.sizes
|
|
288
|
-
and generic_pointing_xds.sizes["dir"] == 0
|
|
289
|
-
):
|
|
290
|
-
# When some direction variables are "empty" but some are populated properly
|
|
291
|
-
if "dim_2" in generic_pointing_xds[key].sizes:
|
|
292
|
-
data_var_data = xr.DataArray(
|
|
293
|
-
generic_pointing_xds[key].values,
|
|
294
|
-
dims=generic_pointing_xds[key].dims,
|
|
295
|
-
)
|
|
296
|
-
else:
|
|
297
|
-
shape = tuple(
|
|
298
|
-
generic_pointing_xds.sizes[dim]
|
|
299
|
-
for dim in ["TIME", "ANTENNA_ID"]
|
|
300
|
-
) + (2,)
|
|
301
|
-
data_var_data = xr.DataArray(
|
|
302
|
-
np.full(shape, np.nan),
|
|
303
|
-
dims=generic_pointing_xds[key].dims,
|
|
304
|
-
)
|
|
305
|
-
else:
|
|
306
|
-
data_var_data = generic_pointing_xds[key].data
|
|
307
|
-
|
|
308
|
-
pointing_xds[data_var_name] = xr.DataArray(
|
|
309
|
-
data_var_data, dims=data_variable_dims[key]
|
|
310
|
-
)
|
|
311
|
-
|
|
312
|
-
msv4_measure = column_description_casacore_to_msv4_measure(
|
|
313
|
-
pointing_column_descriptions[key.upper()]
|
|
314
|
-
)
|
|
315
|
-
if msv4_measure:
|
|
316
|
-
pointing_xds[data_var_name].attrs.update(msv4_measure)
|
|
317
|
-
|
|
318
268
|
coords = {
|
|
319
|
-
"time": generic_pointing_xds["TIME"].values,
|
|
320
269
|
"antenna_name": ant_xds_name_ids.sel(
|
|
321
270
|
antenna_id=generic_pointing_xds["ANTENNA_ID"]
|
|
322
271
|
).data,
|
|
323
272
|
"sky_dir_label": ["ra", "dec"],
|
|
324
273
|
}
|
|
325
274
|
pointing_xds = pointing_xds.assign_coords(coords)
|
|
275
|
+
pointing_xds = convert_generic_xds_to_xradio_schema(
|
|
276
|
+
generic_pointing_xds, pointing_xds, to_new_data_variables, to_new_coords
|
|
277
|
+
)
|
|
326
278
|
|
|
327
|
-
#
|
|
328
|
-
pointing_xds["time"].attrs.update({"units": ["s"], "type": "quantity"})
|
|
329
|
-
|
|
279
|
+
# Add attributes specific to pointing_xds
|
|
330
280
|
if "TRACKING" in generic_pointing_xds.data_vars:
|
|
331
281
|
pointing_xds.attrs["tracking"] = generic_pointing_xds.data_vars[
|
|
332
282
|
"TRACKING"
|
|
@@ -335,6 +285,10 @@ def create_pointing_xds(
|
|
|
335
285
|
# Move target from data_vars to attributes?
|
|
336
286
|
move_target_as_attr = False
|
|
337
287
|
if move_target_as_attr:
|
|
288
|
+
pointing_column_descriptions = generic_pointing_xds.attrs["other"]["msv2"][
|
|
289
|
+
"ctds_attrs"
|
|
290
|
+
]["column_descriptions"]
|
|
291
|
+
|
|
338
292
|
target = generic_pointing_xds.data_vars["TARGET"]
|
|
339
293
|
pointing_xds.attrs["target"] = {
|
|
340
294
|
"dims": ["sky_dir_label"],
|
|
@@ -348,4 +302,5 @@ def create_pointing_xds(
|
|
|
348
302
|
pointing_xds = interpolate_to_time(pointing_xds, interp_time, "pointing_xds")
|
|
349
303
|
|
|
350
304
|
logger.debug(f"create_pointing_xds() execution time {time.time() - start:0.2f} s")
|
|
305
|
+
|
|
351
306
|
return pointing_xds
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
from importlib.metadata import version
|
|
2
|
-
import
|
|
2
|
+
import toolviper.utils.logger as logger, multiprocessing, psutil
|
|
3
3
|
from typing import Any, Dict, List, Tuple, Union
|
|
4
4
|
|
|
5
5
|
import numpy as np
|
|
@@ -7,6 +7,7 @@ import xarray as xr
|
|
|
7
7
|
|
|
8
8
|
from .cds import CASAVisSet
|
|
9
9
|
from .stokes_types import stokes_types
|
|
10
|
+
from ...._utils.common import get_pad_value
|
|
10
11
|
|
|
11
12
|
|
|
12
13
|
def make_coords(
|
|
@@ -195,7 +196,8 @@ def expand_xds(xds: xr.Dataset) -> xr.Dataset:
|
|
|
195
196
|
except Exception as exc:
|
|
196
197
|
logger.warning(
|
|
197
198
|
f"WARNING: Cannot expand rows to (time, baseline), "
|
|
198
|
-
f"possibly duplicate values in (time, baseline). Exception: {exc}"
|
|
199
|
+
f"possibly duplicate values in (time, baseline). Exception: {exc}."
|
|
200
|
+
f"\nDataset: {txds=}"
|
|
199
201
|
)
|
|
200
202
|
txds = xds.copy()
|
|
201
203
|
|
|
@@ -214,10 +216,8 @@ def flatten_xds(xds: xr.Dataset) -> xr.Dataset:
|
|
|
214
216
|
Returns
|
|
215
217
|
-------
|
|
216
218
|
xr.Dataset
|
|
219
|
+
Dataset in flat form (back to 'row' dimension as read by casacore tables)
|
|
217
220
|
"""
|
|
218
|
-
# known invalid cast warning when casting to integer
|
|
219
|
-
with np.errstate(invalid="ignore"):
|
|
220
|
-
nan_int = np.array([np.nan]).astype("int32")[0]
|
|
221
221
|
txds = xds.copy()
|
|
222
222
|
|
|
223
223
|
# flatten the time x baseline dimensions of main table
|
|
@@ -226,14 +226,12 @@ def flatten_xds(xds: xr.Dataset) -> xr.Dataset:
|
|
|
226
226
|
# compute for issue https://github.com/hainegroup/oceanspy/issues/332
|
|
227
227
|
# drop=True silently does compute (or at least used to)
|
|
228
228
|
|
|
229
|
-
|
|
230
|
-
# txds = txds.where(
|
|
231
|
-
# ((txds.STATE_ID != nan_int) & (txds.FIELD_ID != nan_int)).compute(),
|
|
232
|
-
# drop=True,
|
|
233
|
-
# ) # .unify_chunks()
|
|
234
|
-
|
|
229
|
+
fill_value_int32 = get_pad_value(np.int32)
|
|
235
230
|
txds = txds.where(
|
|
236
|
-
|
|
231
|
+
(
|
|
232
|
+
(txds.STATE_ID != fill_value_int32)
|
|
233
|
+
& (txds.FIELD_ID != fill_value_int32)
|
|
234
|
+
).compute(),
|
|
237
235
|
drop=True,
|
|
238
236
|
) # .unify_chunks()
|
|
239
237
|
|
xradio/vis/_vis_utils/ms.py
CHANGED
xradio/vis/_vis_utils/zarr.py
CHANGED
|
@@ -3,7 +3,7 @@ from pathlib import Path
|
|
|
3
3
|
from typing import Dict, Union
|
|
4
4
|
|
|
5
5
|
import zarr
|
|
6
|
-
import
|
|
6
|
+
import toolviper.utils.logger as logger
|
|
7
7
|
|
|
8
8
|
from ._utils.cds import CASAVisSet
|
|
9
9
|
from ._zarr.read import read_part_keys, read_partitions, read_subtables
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: xradio
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.40
|
|
4
4
|
Summary: Xarray Radio Astronomy Data IO
|
|
5
5
|
Author-email: Jan-Willem Steeb <jsteeb@nrao.edu>
|
|
6
6
|
License: BSD 3-Clause License
|
|
@@ -44,9 +44,10 @@ Requires-Dist: astropy
|
|
|
44
44
|
Requires-Dist: dask
|
|
45
45
|
Requires-Dist: distributed
|
|
46
46
|
Requires-Dist: graphviper
|
|
47
|
+
Requires-Dist: toolviper
|
|
47
48
|
Requires-Dist: matplotlib
|
|
48
49
|
Requires-Dist: numba >=0.57.0
|
|
49
|
-
Requires-Dist: numpy
|
|
50
|
+
Requires-Dist: numpy
|
|
50
51
|
Requires-Dist: prettytable
|
|
51
52
|
Requires-Dist: pytest
|
|
52
53
|
Requires-Dist: pytest-cov
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
xradio/__init__.py,sha256=
|
|
1
|
+
xradio/__init__.py,sha256=WHBhQWQie3YQqfIxQBL3LKiKuUcN7ZL7sPMEcdWOp5E,382
|
|
2
2
|
xradio/_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
3
|
-
xradio/_utils/common.py,sha256=
|
|
3
|
+
xradio/_utils/common.py,sha256=GY_fjet9wLhnx9XUINTq6ONBoceyLWRXHXlrNla3j3o,3726
|
|
4
4
|
xradio/_utils/list_and_array.py,sha256=_wznOiHra1pHrWNdtQPXhk-TAXJ8qb9fTGE0czQNpIo,2802
|
|
5
|
-
xradio/_utils/schema.py,sha256=
|
|
5
|
+
xradio/_utils/schema.py,sha256=tdGyvQTd-rQivtkzmH9-1f7KMIB9KeSf57EN2v7Mw_o,6855
|
|
6
6
|
xradio/_utils/_casacore/tables.py,sha256=aq6E_4RRAHdTBCwMKrVil1cWhFU2O980DNH9IlRKXLw,1280
|
|
7
7
|
xradio/_utils/zarr/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
8
|
xradio/_utils/zarr/common.py,sha256=egj3Zma0BUK0msOBDozMa-62rHrcxrjCNE5XkkZUq70,5332
|
|
@@ -16,13 +16,13 @@ xradio/image/_util/image_factory.py,sha256=6tPzs20FTm2wEshHc1xqtTV7D0TbKxGLUKAVt
|
|
|
16
16
|
xradio/image/_util/zarr.py,sha256=lhQqVRC1GEWClG3zRbuDr2IlQBfXeDqaLUJIN-MVMxA,1652
|
|
17
17
|
xradio/image/_util/_casacore/__init__.py,sha256=OlsiRE40o1jSbBI4khgQQzgfDYbAlOMKIhO4UFlbGhg,41
|
|
18
18
|
xradio/image/_util/_casacore/common.py,sha256=ky999eTCWta8w-uIs-7P7rPhZRLuh9yTuQXAxPvaPm4,1579
|
|
19
|
-
xradio/image/_util/_casacore/xds_from_casacore.py,sha256=
|
|
19
|
+
xradio/image/_util/_casacore/xds_from_casacore.py,sha256=4puiE2_1-7Y1-0yK7L8FYyDp5Ab8hEhDjLvoXiGMx9E,42561
|
|
20
20
|
xradio/image/_util/_casacore/xds_to_casacore.py,sha256=P6c-yoOjuVQkm07ApA7FFKfje4aPwV-MsRFKaRaPq9I,15338
|
|
21
21
|
xradio/image/_util/_fits/xds_from_fits.py,sha256=kpZ2gdbDQtoUD1DdQZymup5OVnB3e73D66LruLYESsw,28445
|
|
22
22
|
xradio/image/_util/_zarr/common.py,sha256=apMX_bF4Hr3pFGjnDFpp36KgmhTYAPBZquNkjBHrsXk,307
|
|
23
|
-
xradio/image/_util/_zarr/xds_from_zarr.py,sha256=
|
|
23
|
+
xradio/image/_util/_zarr/xds_from_zarr.py,sha256=4b6KHmAcnrhBbCi-Z7e3Lm6l6wziJL1zaNIohmPAYDk,3601
|
|
24
24
|
xradio/image/_util/_zarr/xds_to_zarr.py,sha256=wogXbwX8n3Sl9PHoc3_Y_LBowQsQ-94HZQFZ5NcxUZA,1624
|
|
25
|
-
xradio/image/_util/_zarr/zarr_low_level.py,sha256=
|
|
25
|
+
xradio/image/_util/_zarr/zarr_low_level.py,sha256=_skL70DhtmVjmxRJsfZaospt3rndB9l-2QoHFF1kAKE,12786
|
|
26
26
|
xradio/schema/__init__.py,sha256=EzEMnOtN8G_wdjo8QBRKfq5MrYgfr_nt1pfunlI6i6Q,733
|
|
27
27
|
xradio/schema/bases.py,sha256=5BiE6gAq2xmaJEyiaGbpCSoNek83ly9f0R0Rv1rx9DM,17081
|
|
28
28
|
xradio/schema/check.py,sha256=Quugw5pC-c3c6C_JujSOQSfU_XVFIwymmg8wM0IZtXY,18521
|
|
@@ -31,43 +31,43 @@ xradio/schema/metamodel.py,sha256=WjtW7pAVzcjLRWifRH3sQoOiN6TV810hARpOIz1M_gw,38
|
|
|
31
31
|
xradio/schema/typing.py,sha256=8-o6fZd99kJ4FVdgBYRTIRJ-wDqpcUNXzCTfJvl3TIw,10439
|
|
32
32
|
xradio/vis/__init__.py,sha256=oOaHEnHp799cu72Zwo4WUQdrgFE2JCy-O0XVhaz3C2w,381
|
|
33
33
|
xradio/vis/_processing_set.py,sha256=rvPAU5ksfODDDaJQBzBlZaHQDSDseV6UqCxN2Qa5gxs,12097
|
|
34
|
-
xradio/vis/convert_msv2_to_processing_set.py,sha256=
|
|
34
|
+
xradio/vis/convert_msv2_to_processing_set.py,sha256=x5P9P8BZiK243iFi5kjn3i4gMgaGbozSN1sh1bgyYNI,5906
|
|
35
35
|
xradio/vis/load_processing_set.py,sha256=sQmKG04grVn9jqOrhGc8K6B2-tk65NR0GBW8I8W62cI,5755
|
|
36
|
-
xradio/vis/read_processing_set.py,sha256=
|
|
36
|
+
xradio/vis/read_processing_set.py,sha256=8h1UDXFZqwObs6J6p5ef0FSA9ILW3A6I_6HMbQrbRFc,4170
|
|
37
37
|
xradio/vis/schema.py,sha256=pQelyv3f6uIW12UKT_sB8KETyXpMX8klPeUOpq4dyd8,44858
|
|
38
38
|
xradio/vis/_vis_utils/__init__.py,sha256=Scu6rKJ2SpO8aG7F-xdTZcYfyWx0viV8gFh8E8ur_gI,93
|
|
39
|
-
xradio/vis/_vis_utils/ms.py,sha256=
|
|
40
|
-
xradio/vis/_vis_utils/zarr.py,sha256=
|
|
39
|
+
xradio/vis/_vis_utils/ms.py,sha256=m7cYGZGbdCD3uSPe1BldgRurZ4Y-K3vJr6aePUVyGjE,4310
|
|
40
|
+
xradio/vis/_vis_utils/zarr.py,sha256=d7T7P2rLzHTvoH7jh4FsiqkFq-PHsxRHUhM4mxbZI9M,3947
|
|
41
41
|
xradio/vis/_vis_utils/_ms/chunks.py,sha256=JTPk3il6fk570BjWZMoOAtsbvnLmqPcBv9EPY6A2yOs,2964
|
|
42
|
-
xradio/vis/_vis_utils/_ms/conversion.py,sha256=
|
|
43
|
-
xradio/vis/_vis_utils/_ms/create_antenna_xds.py,sha256=
|
|
44
|
-
xradio/vis/_vis_utils/_ms/create_field_and_source_xds.py,sha256=
|
|
42
|
+
xradio/vis/_vis_utils/_ms/conversion.py,sha256=KTpv5CaPYVsqu3WQI-j4yZiNL3NWn6sev905tAyzq8I,38889
|
|
43
|
+
xradio/vis/_vis_utils/_ms/create_antenna_xds.py,sha256=WMX1Av328Et0o56pTcva5tYSOFlu-mBxhJJHp33rb3k,17181
|
|
44
|
+
xradio/vis/_vis_utils/_ms/create_field_and_source_xds.py,sha256=d_WlX_xA33rpHS8Dhqct8HmczZfteA2W7CHC1EPuZEw,30897
|
|
45
45
|
xradio/vis/_vis_utils/_ms/descr.py,sha256=dYK8mhXxODIh-dfqaOm-YZb7kmoN1N2golX_RFncO94,5215
|
|
46
46
|
xradio/vis/_vis_utils/_ms/msv2_msv3.py,sha256=9AKs2HWly7Ivv_Cjr11dIPGmm33_rtSBoGF9wN5ZwEQ,116
|
|
47
|
-
xradio/vis/_vis_utils/_ms/msv2_to_msv4_meta.py,sha256=
|
|
47
|
+
xradio/vis/_vis_utils/_ms/msv2_to_msv4_meta.py,sha256=gk9gU7g2Lk7dmaiLW8qecOEt574pRtGsCHnUnHXM3D0,1614
|
|
48
48
|
xradio/vis/_vis_utils/_ms/msv4_infos.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
49
|
-
xradio/vis/_vis_utils/_ms/msv4_sub_xdss.py,sha256=
|
|
49
|
+
xradio/vis/_vis_utils/_ms/msv4_sub_xdss.py,sha256=DaLIqC05AZbkdc-XiahesYwfRFyLXNlWW4DoBv72d5I,10649
|
|
50
50
|
xradio/vis/_vis_utils/_ms/optimised_functions.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
51
|
-
xradio/vis/_vis_utils/_ms/partition_queries.py,sha256=
|
|
52
|
-
xradio/vis/_vis_utils/_ms/partitions.py,sha256=
|
|
53
|
-
xradio/vis/_vis_utils/_ms/subtables.py,sha256=
|
|
54
|
-
xradio/vis/_vis_utils/_ms/_tables/load.py,sha256=
|
|
55
|
-
xradio/vis/_vis_utils/_ms/_tables/load_main_table.py,sha256=
|
|
56
|
-
xradio/vis/_vis_utils/_ms/_tables/read.py,sha256=
|
|
57
|
-
xradio/vis/_vis_utils/_ms/_tables/read_main_table.py,sha256=
|
|
58
|
-
xradio/vis/_vis_utils/_ms/_tables/read_subtables.py,sha256=
|
|
51
|
+
xradio/vis/_vis_utils/_ms/partition_queries.py,sha256=hEe3VAzGqtEPg3X4GUhvHaxyesjS6WVy5et421qSdZg,14573
|
|
52
|
+
xradio/vis/_vis_utils/_ms/partitions.py,sha256=_KhRq8bSx2QxuWp9K57fLoLxcU6kvJ35e6wvJ-THbwc,12979
|
|
53
|
+
xradio/vis/_vis_utils/_ms/subtables.py,sha256=6--dh6betXhlxGejzcJJd8bYdiNkl5ppzLoyorofZ6c,3943
|
|
54
|
+
xradio/vis/_vis_utils/_ms/_tables/load.py,sha256=IR3fdKlq8rgH4bHmB1JTtB5gSGuITIvErJEVjUA8rWM,1799
|
|
55
|
+
xradio/vis/_vis_utils/_ms/_tables/load_main_table.py,sha256=_Ov4cs5t06RqfmrAHZoYwpb8K27b7iqjIkCMsJ0TR6o,14798
|
|
56
|
+
xradio/vis/_vis_utils/_ms/_tables/read.py,sha256=ZrDHvUoyiAJu6P_2LIpedZgrDuWBwMi99RdMHrj6wyY,42555
|
|
57
|
+
xradio/vis/_vis_utils/_ms/_tables/read_main_table.py,sha256=8AbNt-AxrhPK3EPRa7xqJXffxzIgfVsv1BDfoVJEXLU,26056
|
|
58
|
+
xradio/vis/_vis_utils/_ms/_tables/read_subtables.py,sha256=JM6pGUQtjQR881u9VqakmbJjppEFq-EVKnEZ14JqnAw,12438
|
|
59
59
|
xradio/vis/_vis_utils/_ms/_tables/table_query.py,sha256=q8EGFf_zIwHcHnvFJOn8hPh8zFZQ3f7BGbXvL3bHad4,555
|
|
60
|
-
xradio/vis/_vis_utils/_ms/_tables/write.py,sha256=
|
|
60
|
+
xradio/vis/_vis_utils/_ms/_tables/write.py,sha256=43XQ-tHhbhex0eUTRknNpPEEOnNR-w1lGCox9WZ9NHE,9540
|
|
61
61
|
xradio/vis/_vis_utils/_ms/_tables/write_exp_api.py,sha256=GDEll8nMwkQGc6vosu4UddFL5_ld7WurRgF9hYFTRmU,15511
|
|
62
62
|
xradio/vis/_vis_utils/_utils/cds.py,sha256=OpvKowSheIthUbcPEv2AoKmxlEt3DqJZS5C1AYh5z10,1179
|
|
63
63
|
xradio/vis/_vis_utils/_utils/partition_attrs.py,sha256=JaePHts_A0EbB4K-0a_uC98RZ2EmfjB9pDSEI11oAwk,3401
|
|
64
64
|
xradio/vis/_vis_utils/_utils/stokes_types.py,sha256=DMa8TmmS7BQ99Xm8c7ZjcRapMtLbrKVxrt4f0qUIOvg,561
|
|
65
|
-
xradio/vis/_vis_utils/_utils/xds_helper.py,sha256=
|
|
65
|
+
xradio/vis/_vis_utils/_utils/xds_helper.py,sha256=jZWR55e574j9NJ4Yf5HdHD7J7QtKdXbXStbblQ66WdE,13218
|
|
66
66
|
xradio/vis/_vis_utils/_zarr/encoding.py,sha256=GENIlThV6a9CUCL6gIGlu9c6NR3OFWNos6mpxZjMwDc,536
|
|
67
|
-
xradio/vis/_vis_utils/_zarr/read.py,sha256=
|
|
68
|
-
xradio/vis/_vis_utils/_zarr/write.py,sha256=
|
|
69
|
-
xradio-0.0.
|
|
70
|
-
xradio-0.0.
|
|
71
|
-
xradio-0.0.
|
|
72
|
-
xradio-0.0.
|
|
73
|
-
xradio-0.0.
|
|
67
|
+
xradio/vis/_vis_utils/_zarr/read.py,sha256=O9DiwD2Gn8WiatQ-Q6WGGSwjsXwFktG4f81lM-mgcSg,7596
|
|
68
|
+
xradio/vis/_vis_utils/_zarr/write.py,sha256=k5IfqtI44Dm4KBDiKFGhL5hN7kwNOulvVHmeP5Mi7N4,10043
|
|
69
|
+
xradio-0.0.40.dist-info/LICENSE.txt,sha256=9CYIJt7riOXo9AD0eXBZviLxo_HebD-2JJI8oiWtzfg,1807
|
|
70
|
+
xradio-0.0.40.dist-info/METADATA,sha256=JSY0V_0Zij1K06oFq-4U_z8rRIZCDFsbBadezmVS-jM,4443
|
|
71
|
+
xradio-0.0.40.dist-info/WHEEL,sha256=cVxcB9AmuTcXqmwrtPhNK88dr7IR_b6qagTj0UvIEbY,91
|
|
72
|
+
xradio-0.0.40.dist-info/top_level.txt,sha256=dQu27fGBZJ2Yk-gW5XeD-dZ76Xa4Xcvk60Vz-dwXp7k,7
|
|
73
|
+
xradio-0.0.40.dist-info/RECORD,,
|
|
File without changes
|