xarray-ms 0.5.0__tar.gz → 0.5.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/.github/workflows/ci.yml +1 -36
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/PKG-INFO +4 -3
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/doc/source/changelog.rst +15 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/doc/source/conf.py +1 -1
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/pyproject.toml +7 -4
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/tests/msv4_test_corpus/test_msv_corpus.py +13 -3
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/tests/test_antenna.py +59 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/tests/test_read.py +63 -0
- xarray_ms-0.5.3/tests/test_table_utils.py +89 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/xarray_ms/backend/msv2/array.py +1 -6
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/xarray_ms/backend/msv2/factories/antenna.py +5 -1
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/xarray_ms/backend/msv2/factories/correlated.py +9 -3
- xarray_ms-0.5.3/xarray_ms/backend/msv2/table_utils.py +53 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/xarray_ms/errors.py +5 -0
- xarray_ms-0.5.0/xarray_ms/backend/msv2/table_utils.py +0 -19
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/.github/ISSUE_TEMPLATE.md +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/.github/PULL_REQUEST_TEMPLATE.md +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/.github/dependabot.yml +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/.github/workflows/pre-commit.yml +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/.github/workflows/readthedocs.yml +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/.gitignore +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/.pre-commit-config.yaml +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/.readthedocs.yaml +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/LICENSE +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/README.rst +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/doc/Makefile +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/doc/make.bat +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/doc/source/api.rst +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/doc/source/index.rst +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/doc/source/install.rst +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/doc/source/introduction.rst +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/doc/source/links.rst +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/doc/source/partitioning.rst +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/doc/source/roadmap.rst +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/doc/source/tutorial.rst +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/hello.txt +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/tests/__init__.py +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/tests/conftest.py +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/tests/msv4_test_corpus/__init__.py +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/tests/msv4_test_corpus/conftest.py +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/tests/test_backend.py +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/tests/test_basic.py +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/tests/test_field_and_source.py +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/tests/test_github.py +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/tests/test_imputation.py +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/tests/test_measures.py +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/tests/test_multiton.py +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/tests/test_structure.py +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/tests/test_utils.py +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/tests/test_zarr_roundtrip.py +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/xarray_ms/__init__.py +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/xarray_ms/backend/msv2/entrypoint.py +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/xarray_ms/backend/msv2/entrypoint_utils.py +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/xarray_ms/backend/msv2/factories/__init__.py +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/xarray_ms/backend/msv2/factories/core.py +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/xarray_ms/backend/msv2/factories/field_and_source.py +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/xarray_ms/backend/msv2/imputation.py +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/xarray_ms/backend/msv2/measures_adapters.py +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/xarray_ms/backend/msv2/measures_encoders.py +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/xarray_ms/backend/msv2/partition.py +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/xarray_ms/backend/msv2/structure.py +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/xarray_ms/casa_types.py +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/xarray_ms/msv4_types.py +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/xarray_ms/multiton.py +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/xarray_ms/query.py +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/xarray_ms/testing/__init__.py +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/xarray_ms/testing/simulator.py +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/xarray_ms/testing/utils.py +0 -0
- {xarray_ms-0.5.0 → xarray_ms-0.5.3}/xarray_ms/utils.py +0 -0
|
@@ -30,42 +30,7 @@ jobs:
|
|
|
30
30
|
fail-fast: false
|
|
31
31
|
matrix:
|
|
32
32
|
os: ["ubuntu-22.04", "macos-13", "macos-14", "macos-15"]
|
|
33
|
-
python-version: ["3.
|
|
34
|
-
is_main_or_release:
|
|
35
|
-
- ${{ contains(github.ref, 'main') || startsWith(github.ref, 'refs/tags')}}
|
|
36
|
-
exclude:
|
|
37
|
-
# Don't test on macos-{13,14} in PR's
|
|
38
|
-
- is_main_or_release: false
|
|
39
|
-
os: "macos-13"
|
|
40
|
-
- is_main_or_release: false
|
|
41
|
-
os: "macos-14"
|
|
42
|
-
# Don't test on 3.10 - 3.12 in PR's
|
|
43
|
-
- is_main_or_release: false
|
|
44
|
-
python-version: "3.10"
|
|
45
|
-
- is_main_or_release: false
|
|
46
|
-
python-version: "3.11"
|
|
47
|
-
- is_main_or_release: false
|
|
48
|
-
python-version: "3.12"
|
|
49
|
-
# Don't test on macos on 3.10 - 3.12 in PRs
|
|
50
|
-
- os: "macos-13"
|
|
51
|
-
python-version: "3.10"
|
|
52
|
-
- os: "macos-13"
|
|
53
|
-
python-version: "3.11"
|
|
54
|
-
- os: "macos-13"
|
|
55
|
-
python-version: "3.12"
|
|
56
|
-
- os: "macos-14"
|
|
57
|
-
python-version: "3.10"
|
|
58
|
-
- os: "macos-14"
|
|
59
|
-
python-version: "3.11"
|
|
60
|
-
- os: "macos-14"
|
|
61
|
-
python-version: "3.12"
|
|
62
|
-
- os: "macos-15"
|
|
63
|
-
python-version: "3.10"
|
|
64
|
-
- os: "macos-15"
|
|
65
|
-
python-version: "3.11"
|
|
66
|
-
- os: "macos-15"
|
|
67
|
-
python-version: "3.12"
|
|
68
|
-
|
|
33
|
+
python-version: ["3.11", "3.12", "3.13", "3.14"]
|
|
69
34
|
|
|
70
35
|
steps:
|
|
71
36
|
- name: Set up Python ${{ matrix.python-version }}
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: xarray-ms
|
|
3
|
-
Version: 0.5.
|
|
3
|
+
Version: 0.5.3
|
|
4
4
|
Summary: xarray MSv4 views over MSv2 Measurement Sets
|
|
5
5
|
Author-email: Simon Perkins <simon.perkins@gmail.com>
|
|
6
6
|
License-File: LICENSE
|
|
7
|
-
Requires-Python: >=3.
|
|
8
|
-
Requires-Dist: arcae<0.
|
|
7
|
+
Requires-Python: >=3.11
|
|
8
|
+
Requires-Dist: arcae<0.6.0,>=0.5.1
|
|
9
9
|
Requires-Dist: cacheout>=0.16.0
|
|
10
10
|
Requires-Dist: typing-extensions>=4.12.2
|
|
11
11
|
Requires-Dist: xarray>=2025.0
|
|
@@ -24,6 +24,7 @@ Requires-Dist: dask>=2024.5.0; extra == 'testing'
|
|
|
24
24
|
Requires-Dist: distributed>=2024.5.0; extra == 'testing'
|
|
25
25
|
Requires-Dist: platformdirs>=4.3.8; extra == 'testing'
|
|
26
26
|
Requires-Dist: pytest>=8.0.0; extra == 'testing'
|
|
27
|
+
Requires-Dist: xradio>=1.1.2; (python_version >= '3.11' and python_version < '3.14') and extra == 'testing'
|
|
27
28
|
Requires-Dist: zarr<3.0.0,>=2.18.3; extra == 'testing'
|
|
28
29
|
Description-Content-Type: text/x-rst
|
|
29
30
|
|
|
@@ -3,6 +3,21 @@
|
|
|
3
3
|
Changelog
|
|
4
4
|
=========
|
|
5
5
|
|
|
6
|
+
0.5.3 (11-05-2026)
|
|
7
|
+
------------------
|
|
8
|
+
* Expand standard msv2 columns supported during reads (:pr:`153`)
|
|
9
|
+
* Handle duplicate antenna name (:pr:`151`)
|
|
10
|
+
* Deprecate python 3.10 support (:pr:`150`)
|
|
11
|
+
* Add python 3.14 to CI testing matrix (:pr:`150`)
|
|
12
|
+
* Test more comprehensively in pull requests (:pr:`150`)
|
|
13
|
+
* Depend on arcae ``>= 0.5.1, < 0.6.0`` (:pr:`149`)
|
|
14
|
+
|
|
15
|
+
0.5.1 (06-03-2026)
|
|
16
|
+
------------------
|
|
17
|
+
* Correct ``BroadcastMSv2Array`` initialisation of ``MSv2Array`` base class
|
|
18
|
+
which resulted in unpickleable ``BroadcastMSv2Array`` instances. (:pr:`147`)
|
|
19
|
+
* Depend on xradio to test xarray-ms MSv4 Datatree compliance (:pr:`148`)
|
|
20
|
+
|
|
6
21
|
0.5.0 (02-03-2026)
|
|
7
22
|
------------------
|
|
8
23
|
* Fix ``VisibilityCoder`` typo (:pr:`145`)
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
project = "xarray-ms"
|
|
12
12
|
copyright = "2024 - 2025 NRF (SARAO) and Rhodes University (RATT) Centre"
|
|
13
13
|
author = "Simon Perkins"
|
|
14
|
-
release = "0.5.
|
|
14
|
+
release = "0.5.3"
|
|
15
15
|
|
|
16
16
|
# -- General configuration ---------------------------------------------------
|
|
17
17
|
# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
|
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "xarray-ms"
|
|
3
|
-
version = "0.5.
|
|
3
|
+
version = "0.5.3"
|
|
4
4
|
description = "xarray MSv4 views over MSv2 Measurement Sets"
|
|
5
5
|
authors = [{name = "Simon Perkins", email = "simon.perkins@gmail.com"}]
|
|
6
6
|
readme = "README.rst"
|
|
7
|
-
requires-python = ">=3.
|
|
7
|
+
requires-python = ">=3.11"
|
|
8
8
|
dependencies = [
|
|
9
9
|
"xarray>=2025.0",
|
|
10
10
|
"cacheout>=0.16.0",
|
|
11
|
-
"arcae>=0.
|
|
11
|
+
"arcae>=0.5.1, < 0.6.0",
|
|
12
12
|
"typing-extensions>=4.12.2",
|
|
13
13
|
]
|
|
14
14
|
|
|
@@ -19,6 +19,7 @@ testing = [
|
|
|
19
19
|
"distributed>=2024.5.0",
|
|
20
20
|
"zarr>=2.18.3, <3.0.0",
|
|
21
21
|
"platformdirs>=4.3.8",
|
|
22
|
+
"xradio>=1.1.2; python_version >= '3.11' and python_version < '3.14'",
|
|
22
23
|
]
|
|
23
24
|
dev = [
|
|
24
25
|
"pre-commit>=3.8.0",
|
|
@@ -54,7 +55,7 @@ extend-select = ["I"]
|
|
|
54
55
|
# github_url = "https://github.com/<user or organization>/<project>/"
|
|
55
56
|
|
|
56
57
|
[tool.tbump.version]
|
|
57
|
-
current = "0.5.
|
|
58
|
+
current = "0.5.3"
|
|
58
59
|
|
|
59
60
|
# Example of a semver regexp.
|
|
60
61
|
# Make sure this matches current_version before
|
|
@@ -76,9 +77,11 @@ tag_template = "{new_version}"
|
|
|
76
77
|
# tbump.toml location.
|
|
77
78
|
[[tool.tbump.file]]
|
|
78
79
|
src = "pyproject.toml"
|
|
80
|
+
search = 'version = "{current_version}"'
|
|
79
81
|
|
|
80
82
|
[[tool.tbump.file]]
|
|
81
83
|
src = "doc/source/conf.py"
|
|
84
|
+
search = 'release = "{current_version}"'
|
|
82
85
|
|
|
83
86
|
# You can specify a list of commands to
|
|
84
87
|
# run after the files have been patched
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import os.path
|
|
2
|
+
import warnings
|
|
2
3
|
|
|
3
4
|
import pytest
|
|
4
5
|
import xarray
|
|
@@ -7,9 +8,15 @@ import xarray_ms # noqa
|
|
|
7
8
|
|
|
8
9
|
# If present, use xradio to perform schema checking
|
|
9
10
|
try:
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
11
|
+
with warnings.catch_warnings():
|
|
12
|
+
warnings.filterwarnings(
|
|
13
|
+
"ignore",
|
|
14
|
+
category=UserWarning,
|
|
15
|
+
message="Could not import the function to convert from MSv2 to MSv4.",
|
|
16
|
+
)
|
|
17
|
+
import xradio # noqa
|
|
18
|
+
import xradio.measurement_set # noqa. Required to register types for check_datatree
|
|
19
|
+
from xradio.schema.check import check_datatree
|
|
13
20
|
except ImportError:
|
|
14
21
|
xradio = None
|
|
15
22
|
|
|
@@ -17,6 +24,9 @@ pmx = pytest.mark.xfail
|
|
|
17
24
|
|
|
18
25
|
|
|
19
26
|
@pytest.mark.msv4_test_corpus
|
|
27
|
+
@pytest.mark.filterwarnings(
|
|
28
|
+
"ignore:Could not import the function to convert from MSv2 to MSv4"
|
|
29
|
+
)
|
|
20
30
|
@pytest.mark.filterwarnings("ignore::xarray_ms.errors.FrameConversionWarning")
|
|
21
31
|
@pytest.mark.filterwarnings("ignore::xarray_ms.errors.ImputedMetadataWarning")
|
|
22
32
|
@pytest.mark.filterwarnings("ignore::xarray_ms.errors.IrregularTimeGridWarning")
|
|
@@ -4,6 +4,8 @@ import pytest
|
|
|
4
4
|
import xarray
|
|
5
5
|
from arcae.lib.arrow_tables import Table, ms_descriptor
|
|
6
6
|
|
|
7
|
+
from xarray_ms.errors import DuplicateAntennaNameWarning
|
|
8
|
+
|
|
7
9
|
NANTENNA = 6
|
|
8
10
|
|
|
9
11
|
|
|
@@ -138,3 +140,60 @@ def test_antenna_feed_join(simmed_ms, auto_corrs):
|
|
|
138
140
|
npt.assert_array_equal(
|
|
139
141
|
p1["antenna_xds"].antenna_name, np.array(["ANTENNA-0", "ANTENNA-2"], dtype=object)
|
|
140
142
|
)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
@pytest.mark.parametrize(
|
|
146
|
+
"simmed_ms",
|
|
147
|
+
[{"name": "dup.ms", "nantenna": 3, "auto_corrs": True}],
|
|
148
|
+
indirect=True,
|
|
149
|
+
)
|
|
150
|
+
def test_duplicate_antenna_names(simmed_ms):
|
|
151
|
+
"""When ANTENNA::NAME contains duplicates, names are made unique and a
|
|
152
|
+
warning is emitted. The names in antenna_xds and the baseline coordinate
|
|
153
|
+
must be consistent with each other."""
|
|
154
|
+
|
|
155
|
+
# Overwrite ANTENNA::NAME so that antenna 0 and antenna 1 share a name.
|
|
156
|
+
with Table.from_filename(f"{simmed_ms}::ANTENNA") as T:
|
|
157
|
+
T.putcol("NAME", np.asarray(["SAME", "SAME", "OTHER"]))
|
|
158
|
+
|
|
159
|
+
with pytest.warns(DuplicateAntennaNameWarning, match="SAME"):
|
|
160
|
+
dt = xarray.open_datatree(simmed_ms, auto_corrs=True).load()
|
|
161
|
+
|
|
162
|
+
partition = dt[list(dt.children)[0]]
|
|
163
|
+
antenna_xds = partition["antenna_xds"]
|
|
164
|
+
|
|
165
|
+
# All antenna_name coordinate values must be unique
|
|
166
|
+
ant_names = antenna_xds.antenna_name.values
|
|
167
|
+
assert len(ant_names) == len(set(ant_names)), "antenna_name coordinate has duplicates"
|
|
168
|
+
|
|
169
|
+
# Duplicated base names get -N suffixes; unique names are unchanged
|
|
170
|
+
assert "SAME-1" in ant_names
|
|
171
|
+
assert "SAME-2" in ant_names
|
|
172
|
+
assert "OTHER" in ant_names
|
|
173
|
+
|
|
174
|
+
# correlated dataset baseline_antenna{i}_name maps
|
|
175
|
+
# appropriately to the antenna dataset antenna names.
|
|
176
|
+
concat_ant_names = np.concatenate(
|
|
177
|
+
[partition.baseline_antenna1_name, partition.baseline_antenna2_name]
|
|
178
|
+
)
|
|
179
|
+
unique_ant_names, inv = np.unique(concat_ant_names, return_inverse=True)
|
|
180
|
+
_, lookup = np.where(unique_ant_names[:, None] == ant_names[None, :])
|
|
181
|
+
ant1 = lookup[inv[: len(inv) // 2]]
|
|
182
|
+
ant2 = lookup[inv[len(inv) // 2 :]]
|
|
183
|
+
|
|
184
|
+
np.testing.assert_array_equal(partition.baseline_antenna1_name, ant_names[ant1])
|
|
185
|
+
np.testing.assert_array_equal(partition.baseline_antenna2_name, ant_names[ant2])
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
@pytest.mark.parametrize(
|
|
189
|
+
"simmed_ms",
|
|
190
|
+
[{"name": "nodup.ms", "nantenna": 3, "auto_corrs": True}],
|
|
191
|
+
indirect=True,
|
|
192
|
+
)
|
|
193
|
+
def test_unique_antenna_names_no_warning(simmed_ms):
|
|
194
|
+
"""When all antenna names are already unique no warning should be emitted."""
|
|
195
|
+
import warnings
|
|
196
|
+
|
|
197
|
+
with warnings.catch_warnings():
|
|
198
|
+
warnings.simplefilter("error", DuplicateAntennaNameWarning)
|
|
199
|
+
xarray.open_datatree(simmed_ms, auto_corrs=True).load()
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import pickle
|
|
1
2
|
from contextlib import nullcontext
|
|
2
3
|
from functools import reduce
|
|
3
4
|
from operator import mul
|
|
@@ -296,6 +297,27 @@ def test_low_resolution_read(simmed_ms):
|
|
|
296
297
|
)
|
|
297
298
|
|
|
298
299
|
|
|
300
|
+
@pytest.mark.parametrize(
|
|
301
|
+
"simmed_ms",
|
|
302
|
+
[
|
|
303
|
+
{
|
|
304
|
+
"name": "backend.ms",
|
|
305
|
+
"data_description": [(8, ["XX", "XY", "YX", "YY"]), (4, ["RR", "LL"])],
|
|
306
|
+
"table_desc": {"__remove_columns__": ["WEIGHT_SPECTRUM"]},
|
|
307
|
+
"transform_data": _remove_weight_spectrum_add_weight,
|
|
308
|
+
}
|
|
309
|
+
],
|
|
310
|
+
indirect=True,
|
|
311
|
+
)
|
|
312
|
+
def test_datatree_pickleable(simmed_ms):
|
|
313
|
+
"""Test that the resulting datatree is pickleable.
|
|
314
|
+
|
|
315
|
+
WEIGHT_SPECTRUM is replaced with WEIGHT to test
|
|
316
|
+
https://github.com/ratt-ru/xarray-ms/pull/146"""
|
|
317
|
+
dt = xarray.open_datatree(simmed_ms, auto_corrs=True)
|
|
318
|
+
xarray.testing.assert_identical(dt, pickle.loads(pickle.dumps(dt)))
|
|
319
|
+
|
|
320
|
+
|
|
299
321
|
@pytest.mark.parametrize(
|
|
300
322
|
"simmed_ms",
|
|
301
323
|
[
|
|
@@ -393,6 +415,30 @@ NONSTANDARD_TABLE_DESC = {
|
|
|
393
415
|
# 'shape': ..., # Variably shaped
|
|
394
416
|
"valueType": "COMPLEX",
|
|
395
417
|
},
|
|
418
|
+
"MODEL_DATA": {
|
|
419
|
+
"_c_order": True,
|
|
420
|
+
"comment": "MODEL_DATA column",
|
|
421
|
+
"dataManagerGroup": "StandardStMan",
|
|
422
|
+
"dataManagerType": "StandardStMan",
|
|
423
|
+
"keywords": {},
|
|
424
|
+
"maxlen": 0,
|
|
425
|
+
"ndim": 2,
|
|
426
|
+
"option": 0,
|
|
427
|
+
# 'shape': ..., # Variably shaped
|
|
428
|
+
"valueType": "COMPLEX",
|
|
429
|
+
},
|
|
430
|
+
"FLOAT_DATA": {
|
|
431
|
+
"_c_order": True,
|
|
432
|
+
"comment": "FLOAT_DATA column",
|
|
433
|
+
"dataManagerGroup": "StandardStMan",
|
|
434
|
+
"dataManagerType": "StandardStMan",
|
|
435
|
+
"keywords": {},
|
|
436
|
+
"maxlen": 0,
|
|
437
|
+
"ndim": 2,
|
|
438
|
+
"option": 0,
|
|
439
|
+
# 'shape': ..., # Variably shaped
|
|
440
|
+
"valueType": "FLOAT",
|
|
441
|
+
},
|
|
396
442
|
"CORRELATED_WEIGHT_SPECTRUM": {
|
|
397
443
|
"_c_order": True,
|
|
398
444
|
"comment": "CORRELATED_WEIGHT_SPECTRUM column",
|
|
@@ -410,6 +456,8 @@ NONSTANDARD_TABLE_DESC = {
|
|
|
410
456
|
|
|
411
457
|
def _add_non_standard_columns(chunk_desc, data_dict):
|
|
412
458
|
data_dict["CORRELATED_DATA"] = data_dict["DATA"]
|
|
459
|
+
data_dict["MODEL_DATA"] = data_dict["DATA"]
|
|
460
|
+
data_dict["FLOAT_DATA"] = (data_dict["DATA"][0], data_dict["DATA"][1].real)
|
|
413
461
|
return data_dict
|
|
414
462
|
|
|
415
463
|
|
|
@@ -441,3 +489,18 @@ def test_additional_columns(simmed_ms):
|
|
|
441
489
|
"polarization",
|
|
442
490
|
)
|
|
443
491
|
assert node["CORRELATED_DATA"].equals(node["VISIBILITY"])
|
|
492
|
+
assert node["FLOAT_DATA"].dims == (
|
|
493
|
+
"time",
|
|
494
|
+
"baseline_id",
|
|
495
|
+
"frequency",
|
|
496
|
+
"polarization",
|
|
497
|
+
)
|
|
498
|
+
assert node["FLOAT_DATA"].equals(node["VISIBILITY"].real)
|
|
499
|
+
|
|
500
|
+
assert node["MODEL_DATA"].dims == (
|
|
501
|
+
"time",
|
|
502
|
+
"baseline_id",
|
|
503
|
+
"frequency",
|
|
504
|
+
"polarization",
|
|
505
|
+
)
|
|
506
|
+
assert node["MODEL_DATA"].equals(node["VISIBILITY"])
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import warnings
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
import numpy.testing as npt
|
|
5
|
+
import pytest
|
|
6
|
+
|
|
7
|
+
from xarray_ms.backend.msv2.table_utils import unique_antenna_names
|
|
8
|
+
from xarray_ms.errors import DuplicateAntennaNameWarning
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def test_all_unique_no_change():
|
|
12
|
+
names = np.array(["A", "B", "C"])
|
|
13
|
+
result = unique_antenna_names(names)
|
|
14
|
+
npt.assert_array_equal(result, names)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def test_all_unique_no_warning():
|
|
18
|
+
names = np.array(["A", "B", "C"])
|
|
19
|
+
with warnings.catch_warnings():
|
|
20
|
+
warnings.simplefilter("error", DuplicateAntennaNameWarning)
|
|
21
|
+
unique_antenna_names(names) # should not raise
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def test_two_duplicates():
|
|
25
|
+
names = np.array(["A", "A", "B"])
|
|
26
|
+
with pytest.warns(DuplicateAntennaNameWarning):
|
|
27
|
+
result = unique_antenna_names(names)
|
|
28
|
+
npt.assert_array_equal(result, ["A-1", "A-2", "B"])
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def test_triple_duplicate():
|
|
32
|
+
names = np.array(["A", "A", "A"])
|
|
33
|
+
with pytest.warns(DuplicateAntennaNameWarning):
|
|
34
|
+
result = unique_antenna_names(names)
|
|
35
|
+
npt.assert_array_equal(result, ["A-1", "A-2", "A-3"])
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def test_non_contiguous_duplicates():
|
|
39
|
+
names = np.array(["A", "B", "A", "C", "A"])
|
|
40
|
+
with pytest.warns(DuplicateAntennaNameWarning):
|
|
41
|
+
result = unique_antenna_names(names)
|
|
42
|
+
npt.assert_array_equal(result, ["A-1", "B", "A-2", "C", "A-3"])
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def test_unique_name_unchanged_when_mixed():
|
|
46
|
+
names = np.array(["X", "Y", "X"])
|
|
47
|
+
with pytest.warns(DuplicateAntennaNameWarning):
|
|
48
|
+
result = unique_antenna_names(names)
|
|
49
|
+
npt.assert_array_equal(result, ["X-1", "Y", "X-2"])
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def test_underscore_in_base_name():
|
|
53
|
+
"""Dash separator avoids collision with names already containing underscores."""
|
|
54
|
+
names = np.array(["ANTENNA_1", "ANTENNA_1"])
|
|
55
|
+
with pytest.warns(DuplicateAntennaNameWarning):
|
|
56
|
+
result = unique_antenna_names(names)
|
|
57
|
+
npt.assert_array_equal(result, ["ANTENNA_1-1", "ANTENNA_1-2"])
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def test_empty_array():
|
|
61
|
+
names = np.array([], dtype=str)
|
|
62
|
+
result = unique_antenna_names(names)
|
|
63
|
+
npt.assert_array_equal(result, [])
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def test_single_element():
|
|
67
|
+
names = np.array(["A"])
|
|
68
|
+
result = unique_antenna_names(names)
|
|
69
|
+
npt.assert_array_equal(result, ["A"])
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def test_warning_raised_on_duplicates():
|
|
73
|
+
names = np.array(["A", "A", "B"])
|
|
74
|
+
with pytest.warns(DuplicateAntennaNameWarning, match="Duplicate antenna names"):
|
|
75
|
+
unique_antenna_names(names)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def test_warning_lists_duplicate_names():
|
|
79
|
+
names = np.array(["FOO", "BAR", "FOO"])
|
|
80
|
+
with pytest.warns(DuplicateAntennaNameWarning, match="FOO"):
|
|
81
|
+
unique_antenna_names(names)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def test_result_is_always_unique():
|
|
85
|
+
"""Output names must all be distinct regardless of input."""
|
|
86
|
+
names = np.array(["A", "A", "B", "B", "C"])
|
|
87
|
+
with pytest.warns(DuplicateAntennaNameWarning):
|
|
88
|
+
result = unique_antenna_names(names)
|
|
89
|
+
assert len(set(result.tolist())) == len(result)
|
|
@@ -160,7 +160,6 @@ class BroadcastMSv2Array(MSv2Array):
|
|
|
160
160
|
|
|
161
161
|
_low_res_array: MSv2Array
|
|
162
162
|
_low_res_index: Tuple[slice | npt.NDArray | None, ...]
|
|
163
|
-
shape: Tuple[int, ...]
|
|
164
163
|
|
|
165
164
|
def __init__(
|
|
166
165
|
self,
|
|
@@ -168,13 +167,9 @@ class BroadcastMSv2Array(MSv2Array):
|
|
|
168
167
|
low_res_index: Tuple[slice | npt.NDArray | None, ...],
|
|
169
168
|
high_res_shape: Tuple[int, ...],
|
|
170
169
|
):
|
|
170
|
+
super().__init__(high_res_shape, low_res_array.dtype)
|
|
171
171
|
self._low_res_array = low_res_array
|
|
172
172
|
self._low_res_index = low_res_index
|
|
173
|
-
self.shape = high_res_shape
|
|
174
|
-
|
|
175
|
-
@property
|
|
176
|
-
def dtype(self):
|
|
177
|
-
return self._low_res_array.dtype
|
|
178
173
|
|
|
179
174
|
@property
|
|
180
175
|
def transform(self) -> TransformerT | None:
|
|
@@ -4,6 +4,7 @@ from xarray import Dataset, Variable
|
|
|
4
4
|
from xarray_ms.backend.msv2.factories.core import DatasetFactory
|
|
5
5
|
from xarray_ms.backend.msv2.imputation import maybe_impute_observation_table
|
|
6
6
|
from xarray_ms.backend.msv2.measures_encoders import MSv2CoderFactory
|
|
7
|
+
from xarray_ms.backend.msv2.table_utils import unique_antenna_names
|
|
7
8
|
from xarray_ms.errors import InvalidMeasurementSet
|
|
8
9
|
|
|
9
10
|
RELOCATABLE_ARRAY = {"ALMA", "VLA", "NOEMA", "EVLA"}
|
|
@@ -53,7 +54,10 @@ class AntennaFactory(DatasetFactory):
|
|
|
53
54
|
)
|
|
54
55
|
|
|
55
56
|
ant_coder_factory = MSv2CoderFactory.from_arrow_table(filtered_ants)
|
|
56
|
-
|
|
57
|
+
# Deduplicate against the full ANTENNA table so suffix assignments are
|
|
58
|
+
# consistent with those produced in the correlated dataset factory.
|
|
59
|
+
all_ant_names = unique_antenna_names(ants["NAME"].to_numpy().astype(str))
|
|
60
|
+
antenna_names = all_ant_names[feed_ant_id[mask]]
|
|
57
61
|
telescope_names = np.asarray([telescope_name] * len(antenna_names), dtype=str)
|
|
58
62
|
position = pac.list_flatten(filtered_ants["POSITION"]).to_numpy().reshape(-1, 3)
|
|
59
63
|
diameter = filtered_ants["DISH_DIAMETER"].to_numpy()
|
|
@@ -26,6 +26,7 @@ from xarray_ms.backend.msv2.measures_encoders import (
|
|
|
26
26
|
VisibilityCoder,
|
|
27
27
|
)
|
|
28
28
|
from xarray_ms.backend.msv2.structure import MSv2StructureFactory, PartitionKeyT
|
|
29
|
+
from xarray_ms.backend.msv2.table_utils import unique_antenna_names
|
|
29
30
|
from xarray_ms.casa_types import ColumnDesc, Polarisations
|
|
30
31
|
from xarray_ms.errors import (
|
|
31
32
|
ColumnShapeImputationWarning,
|
|
@@ -175,11 +176,16 @@ class CorrelatedFactory(DatasetFactory):
|
|
|
175
176
|
from arcae.lib.arrow_tables import ms_descriptor
|
|
176
177
|
|
|
177
178
|
# Ignore all standard msv2 columns, except
|
|
178
|
-
# for CORRECTED_DATA and
|
|
179
|
+
# for CORRECTED_DATA, CORRECTED_WEIGHT_SPECTRUM, FLOAT_DATA and MODEL_DATA
|
|
179
180
|
ignored_msv2_column_set = {
|
|
180
181
|
c for c in ms_descriptor("MAIN", complete=True).keys() if not c.startswith("_")
|
|
181
182
|
}
|
|
182
|
-
ignored_msv2_column_set -= {
|
|
183
|
+
ignored_msv2_column_set -= {
|
|
184
|
+
"CORRECTED_DATA",
|
|
185
|
+
"CORRECTED_WEIGHT_SPECTRUM",
|
|
186
|
+
"MODEL_DATA",
|
|
187
|
+
"FLOAT_DATA",
|
|
188
|
+
}
|
|
183
189
|
remaining_columns = set(self._ms_factory.instance.columns())
|
|
184
190
|
remaining_columns -= ignored_msv2_column_set
|
|
185
191
|
remaining_columns -= processed_columns
|
|
@@ -266,7 +272,7 @@ class CorrelatedFactory(DatasetFactory):
|
|
|
266
272
|
assert (partition.nbl,) == ant1.shape
|
|
267
273
|
|
|
268
274
|
antenna = self._subtable_factories["ANTENNA"].instance
|
|
269
|
-
ant_names = antenna["NAME"].to_numpy().astype(str)
|
|
275
|
+
ant_names = unique_antenna_names(antenna["NAME"].to_numpy().astype(str))
|
|
270
276
|
ant1_names = ant_names[ant1]
|
|
271
277
|
ant2_names = ant_names[ant2]
|
|
272
278
|
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import warnings
|
|
3
|
+
from typing import Any, Dict
|
|
4
|
+
|
|
5
|
+
import numpy as np
|
|
6
|
+
import pyarrow as pa
|
|
7
|
+
|
|
8
|
+
from xarray_ms.errors import DuplicateAntennaNameWarning
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def unique_antenna_names(names: np.ndarray) -> np.ndarray:
|
|
12
|
+
"""Return antenna names with duplicates made unique by appending -N suffixes.
|
|
13
|
+
|
|
14
|
+
All occurrences of a duplicated name are renamed: the first gets ``-1``,
|
|
15
|
+
the second ``-2``, and so on. Names that are already unique are left
|
|
16
|
+
unchanged. A :class:`~xarray_ms.errors.DuplicateAntennaNameWarning` is
|
|
17
|
+
emitted when any renaming is performed.
|
|
18
|
+
"""
|
|
19
|
+
unique, counts = np.unique(names, return_counts=True)
|
|
20
|
+
if not (duplicates := set(unique[counts > 1].tolist())):
|
|
21
|
+
return names
|
|
22
|
+
|
|
23
|
+
warnings.warn(
|
|
24
|
+
f"Duplicate antenna names detected in the ANTENNA table: "
|
|
25
|
+
f"{sorted(duplicates)}. "
|
|
26
|
+
f"A numeric suffix will be appended to make names unique.",
|
|
27
|
+
DuplicateAntennaNameWarning,
|
|
28
|
+
stacklevel=2,
|
|
29
|
+
)
|
|
30
|
+
# Build as a Python list to avoid numpy fixed-width string truncation,
|
|
31
|
+
# then convert back to an array with a wide enough dtype.
|
|
32
|
+
result = names.tolist()
|
|
33
|
+
counters: Dict[str, int] = {}
|
|
34
|
+
for i, name in enumerate(result):
|
|
35
|
+
if name in duplicates:
|
|
36
|
+
counters[name] = counters.get(name, 0) + 1
|
|
37
|
+
result[i] = f"{name}-{counters[name]}"
|
|
38
|
+
return np.array(result, dtype=str)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def extract_table_desc(table: pa.Table) -> Dict[str, Any]:
|
|
42
|
+
"""Extract the CASA table descriptor stored in an Arrow table constructed by arcae"""
|
|
43
|
+
try:
|
|
44
|
+
arcae_metadata = table.schema.metadata[b"__arcae_metadata__"]
|
|
45
|
+
except KeyError:
|
|
46
|
+
raise KeyError("__arcae_metadata__ was not present in the table metadata")
|
|
47
|
+
|
|
48
|
+
try:
|
|
49
|
+
return json.loads(arcae_metadata)["__casa_descriptor__"]
|
|
50
|
+
except KeyError:
|
|
51
|
+
raise KeyError(
|
|
52
|
+
f"arcae metadata {arcae_metadata} does not contain a __casa_descriptor__"
|
|
53
|
+
)
|
|
@@ -30,6 +30,11 @@ class ImputedMetadataWarning(MissingMetadataWarning):
|
|
|
30
30
|
if the original metadata is missing"""
|
|
31
31
|
|
|
32
32
|
|
|
33
|
+
class DuplicateAntennaNameWarning(UserWarning):
|
|
34
|
+
"""Warning raised when duplicate antenna names are found in the ANTENNA table
|
|
35
|
+
and are made unique by appending a numeric suffix"""
|
|
36
|
+
|
|
37
|
+
|
|
33
38
|
class FrameConversionWarning(UserWarning):
|
|
34
39
|
"""Warning raised if there's no defined conversion from a CASA
|
|
35
40
|
to astropy reference frame"""
|
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
import json
|
|
2
|
-
from typing import Any, Dict
|
|
3
|
-
|
|
4
|
-
import pyarrow as pa
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
def extract_table_desc(table: pa.Table) -> Dict[str, Any]:
|
|
8
|
-
"""Extract the CASA table descriptor stored in an Arrow table constructed by arcae"""
|
|
9
|
-
try:
|
|
10
|
-
arcae_metadata = table.schema.metadata[b"__arcae_metadata__"]
|
|
11
|
-
except KeyError:
|
|
12
|
-
raise KeyError("__arcae_metadata__ was not present in the table metadata")
|
|
13
|
-
|
|
14
|
-
try:
|
|
15
|
-
return json.loads(arcae_metadata)["__casa_descriptor__"]
|
|
16
|
-
except KeyError:
|
|
17
|
-
raise KeyError(
|
|
18
|
-
f"arcae metadata {arcae_metadata} does not contain a __casa_descriptor__"
|
|
19
|
-
)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|