dask-ms-2 0.2.33__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dask_ms_2-0.2.33.dist-info/METADATA +185 -0
- dask_ms_2-0.2.33.dist-info/RECORD +108 -0
- dask_ms_2-0.2.33.dist-info/WHEEL +4 -0
- dask_ms_2-0.2.33.dist-info/entry_points.txt +3 -0
- dask_ms_2-0.2.33.dist-info/licenses/AUTHORS.rst +13 -0
- dask_ms_2-0.2.33.dist-info/licenses/LICENSE +28 -0
- daskms/__init__.py +27 -0
- daskms/apps/__init__.py +0 -0
- daskms/apps/application.py +15 -0
- daskms/apps/conf/logging.ini +23 -0
- daskms/apps/conftest.py +67 -0
- daskms/apps/convert.py +280 -0
- daskms/apps/entrypoint.py +19 -0
- daskms/apps/formats.py +280 -0
- daskms/apps/fragments.py +63 -0
- daskms/apps/katdal_import.py +75 -0
- daskms/apps/tests/test_chunk_transformer.py +11 -0
- daskms/apps/tests/test_convert.py +61 -0
- daskms/array_api_utils.py +63 -0
- daskms/casacure_backend.py +48 -0
- daskms/columns.py +301 -0
- daskms/config.py +3 -0
- daskms/conftest.py +285 -0
- daskms/constants.py +3 -0
- daskms/dask_ms.py +451 -0
- daskms/dataset.py +536 -0
- daskms/dataset_schema.py +332 -0
- daskms/descriptors/__init__.py +0 -0
- daskms/descriptors/builder.py +202 -0
- daskms/descriptors/builder_factory.py +151 -0
- daskms/descriptors/ms.py +309 -0
- daskms/descriptors/ms_subtable.py +44 -0
- daskms/descriptors/ratt_ms.py +23 -0
- daskms/descriptors/register_default_builders.py +5 -0
- daskms/descriptors/tests/test_builder_factory.py +66 -0
- daskms/descriptors/tests/test_default_builder.py +91 -0
- daskms/descriptors/tests/test_ms_builder.py +110 -0
- daskms/descriptors/tests/test_ms_subtable_builder.py +37 -0
- daskms/example_data.py +124 -0
- daskms/experimental/__init__.py +0 -0
- daskms/experimental/arrow/__init__.py +2 -0
- daskms/experimental/arrow/arrow_schema.py +190 -0
- daskms/experimental/arrow/extension_types.py +207 -0
- daskms/experimental/arrow/reads.py +352 -0
- daskms/experimental/arrow/require_arrow.py +5 -0
- daskms/experimental/arrow/tests/__init__.py +0 -0
- daskms/experimental/arrow/tests/test_extensions.py +65 -0
- daskms/experimental/arrow/tests/test_parquet.py +220 -0
- daskms/experimental/arrow/writes.py +178 -0
- daskms/experimental/fragments/__init__.py +237 -0
- daskms/experimental/fragments/tests/__init__.py +0 -0
- daskms/experimental/fragments/tests/test_fragments.py +346 -0
- daskms/experimental/katdal/__init__.py +1 -0
- daskms/experimental/katdal/constants.py +19 -0
- daskms/experimental/katdal/corr_products.py +68 -0
- daskms/experimental/katdal/katdal_import.py +139 -0
- daskms/experimental/katdal/meerkat_antennas.py +70 -0
- daskms/experimental/katdal/mock_dataset.py +88 -0
- daskms/experimental/katdal/msv2_facade.py +638 -0
- daskms/experimental/katdal/tests/conftest.py +59 -0
- daskms/experimental/katdal/tests/test_chunkstore.py +172 -0
- daskms/experimental/katdal/transpose.py +131 -0
- daskms/experimental/katdal/uvw.py +77 -0
- daskms/experimental/tests/__init__.py +0 -0
- daskms/experimental/tests/test_utils.py +112 -0
- daskms/experimental/utils.py +218 -0
- daskms/experimental/zarr/__init__.py +528 -0
- daskms/experimental/zarr/tests/__init__.py +0 -0
- daskms/experimental/zarr/tests/conftest.py +56 -0
- daskms/experimental/zarr/tests/test_zarr.py +466 -0
- daskms/expressions.py +109 -0
- daskms/fsspec_store.py +189 -0
- daskms/multiton.py +182 -0
- daskms/optimisation.py +349 -0
- daskms/ordering.py +220 -0
- daskms/patterns.py +394 -0
- daskms/query.py +30 -0
- daskms/reads.py +558 -0
- daskms/table.py +30 -0
- daskms/table_executor.py +65 -0
- daskms/table_proxy.py +436 -0
- daskms/table_schemas.py +186 -0
- daskms/testing.py +21 -0
- daskms/tests/__init__.py +0 -0
- daskms/tests/test_array_api_utils.py +265 -0
- daskms/tests/test_columns.py +73 -0
- daskms/tests/test_dataset.py +648 -0
- daskms/tests/test_dataset_keywords.py +119 -0
- daskms/tests/test_dataset_schema.py +53 -0
- daskms/tests/test_executor.py +73 -0
- daskms/tests/test_expression.py +27 -0
- daskms/tests/test_fsspec_store.py +175 -0
- daskms/tests/test_github.py +31 -0
- daskms/tests/test_ms_creation.py +335 -0
- daskms/tests/test_ms_read_and_update.py +766 -0
- daskms/tests/test_multiton_metaclass.py +156 -0
- daskms/tests/test_optimisation.py +139 -0
- daskms/tests/test_optional.py +410 -0
- daskms/tests/test_ordering.py +224 -0
- daskms/tests/test_patterns.py +141 -0
- daskms/tests/test_storage.py +71 -0
- daskms/tests/test_stress.py +32 -0
- daskms/tests/test_table.py +15 -0
- daskms/tests/test_table_proxy.py +236 -0
- daskms/tests/test_table_schemas.py +40 -0
- daskms/tests/test_utils.py +111 -0
- daskms/utils.py +269 -0
- daskms/writes.py +782 -0
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: dask-ms-2
|
|
3
|
+
Version: 0.2.33
|
|
4
|
+
Summary: xarray Dataset from CASA Tables
|
|
5
|
+
Author-email: Simon Perkins <simon.perkins@gmail.com>
|
|
6
|
+
License-Expression: BSD-3-Clause
|
|
7
|
+
License-File: AUTHORS.rst
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Requires-Python: <3.15,>=3.11
|
|
10
|
+
Requires-Dist: appdirs>=1.4.4
|
|
11
|
+
Requires-Dist: cacheout>=0.16.0
|
|
12
|
+
Requires-Dist: dask[array]>=2023.1.1
|
|
13
|
+
Requires-Dist: donfig>=0.8.0
|
|
14
|
+
Requires-Dist: fsspec>=2022.7.0
|
|
15
|
+
Requires-Dist: numpy>=2.0.0
|
|
16
|
+
Provides-Extra: arrow
|
|
17
|
+
Requires-Dist: pandas>=2.1.2; extra == 'arrow'
|
|
18
|
+
Requires-Dist: pyarrow>=14.0.1; extra == 'arrow'
|
|
19
|
+
Provides-Extra: backend
|
|
20
|
+
Requires-Dist: casacure; extra == 'backend'
|
|
21
|
+
Provides-Extra: casacore
|
|
22
|
+
Requires-Dist: python-casacore>=3.7.0; extra == 'casacore'
|
|
23
|
+
Provides-Extra: casacure
|
|
24
|
+
Requires-Dist: casacure; extra == 'casacure'
|
|
25
|
+
Provides-Extra: complete
|
|
26
|
+
Requires-Dist: katdal>=0.23; extra == 'complete'
|
|
27
|
+
Requires-Dist: numcodecs<0.16.0; extra == 'complete'
|
|
28
|
+
Requires-Dist: pandas; extra == 'complete'
|
|
29
|
+
Requires-Dist: pyarrow>=14.0.1; extra == 'complete'
|
|
30
|
+
Requires-Dist: s3fs>=2023.1.0; extra == 'complete'
|
|
31
|
+
Requires-Dist: xarray>=2023.01.0; extra == 'complete'
|
|
32
|
+
Requires-Dist: zarr<3.0.0,>=2.12.0; extra == 'complete'
|
|
33
|
+
Provides-Extra: dev
|
|
34
|
+
Requires-Dist: pre-commit>=4.3.0; extra == 'dev'
|
|
35
|
+
Requires-Dist: ruff>=0.14.0; extra == 'dev'
|
|
36
|
+
Requires-Dist: tbump>=6.11.0; extra == 'dev'
|
|
37
|
+
Provides-Extra: docs
|
|
38
|
+
Requires-Dist: furo>=2025.9.25; extra == 'docs'
|
|
39
|
+
Requires-Dist: numpydoc>=1.9.0; extra == 'docs'
|
|
40
|
+
Requires-Dist: pygments>=2.19.2; extra == 'docs'
|
|
41
|
+
Requires-Dist: sphinx-copybutton>=0.5.2; extra == 'docs'
|
|
42
|
+
Requires-Dist: sphinx>=5.1.1; extra == 'docs'
|
|
43
|
+
Provides-Extra: katdal
|
|
44
|
+
Requires-Dist: katdal>=0.23; extra == 'katdal'
|
|
45
|
+
Requires-Dist: numcodecs<0.16.0; extra == 'katdal'
|
|
46
|
+
Requires-Dist: xarray>=2023.01.0; extra == 'katdal'
|
|
47
|
+
Requires-Dist: zarr<3.0.0,>=2.12.0; extra == 'katdal'
|
|
48
|
+
Provides-Extra: s3
|
|
49
|
+
Requires-Dist: s3fs>=2023.1.0; extra == 's3'
|
|
50
|
+
Provides-Extra: testing
|
|
51
|
+
Requires-Dist: moto[server]>=5.0.0; extra == 'testing'
|
|
52
|
+
Requires-Dist: pytest!=9.1.0,>=8.0.0; extra == 'testing'
|
|
53
|
+
Provides-Extra: xarray
|
|
54
|
+
Requires-Dist: xarray>=2023.01.0; extra == 'xarray'
|
|
55
|
+
Provides-Extra: zarr
|
|
56
|
+
Requires-Dist: numcodecs<0.16.0; extra == 'zarr'
|
|
57
|
+
Requires-Dist: zarr<3.0.0,>=2.12.0; extra == 'zarr'
|
|
58
|
+
Description-Content-Type: text/x-rst
|
|
59
|
+
|
|
60
|
+
================================
|
|
61
|
+
xarray Datasets from CASA Tables
|
|
62
|
+
================================
|
|
63
|
+
|
|
64
|
+
.. image:: https://img.shields.io/pypi/v/dask-ms.svg
|
|
65
|
+
:target: https://pypi.python.org/pypi/dask-ms
|
|
66
|
+
|
|
67
|
+
.. image:: https://github.com/ratt-ru/dask-ms/actions/workflows/ci.yml/badge.svg
|
|
68
|
+
:target: https://github.com/ratt-ru/dask-ms/actions/workflows/ci.yml
|
|
69
|
+
|
|
70
|
+
.. image:: https://readthedocs.org/projects/dask-ms/badge/?version=latest
|
|
71
|
+
:target: https://dask-ms.readthedocs.io/en/latest/?badge=latest
|
|
72
|
+
:alt: Documentation Status
|
|
73
|
+
|
|
74
|
+
Constructs xarray_ ``Datasets`` from CASA Tables via python-casacore_.
|
|
75
|
+
The ``Variables`` contained in the ``Dataset`` are dask_ arrays backed by
|
|
76
|
+
deferred calls to :code:`casacore.tables.table.getcol`.
|
|
77
|
+
|
|
78
|
+
Supports writing ``Variables`` back to the respective column in the Table.
|
|
79
|
+
|
|
80
|
+
The intention behind this package is to support the Measurement Set as
|
|
81
|
+
a data source and sink for the purposes of writing parallel, distributed
|
|
82
|
+
Radio Astronomy algorithms.
|
|
83
|
+
|
|
84
|
+
Installation
|
|
85
|
+
============
|
|
86
|
+
|
|
87
|
+
To install with xarray_ support:
|
|
88
|
+
|
|
89
|
+
.. code-block:: bash
|
|
90
|
+
|
|
91
|
+
$ pip install dask-ms-2[xarray]
|
|
92
|
+
|
|
93
|
+
Without xarray_ similar, but reduced Dataset functionality is replicated
|
|
94
|
+
in dask-ms itself. Expert users may wish to use this option to reduce
|
|
95
|
+
python package dependencies.
|
|
96
|
+
|
|
97
|
+
.. code-block:: bash
|
|
98
|
+
|
|
99
|
+
$ pip install dask-ms-2
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
Documentation
|
|
103
|
+
=============
|
|
104
|
+
|
|
105
|
+
https://dask-ms.readthedocs.io
|
|
106
|
+
|
|
107
|
+
Gitter Page
|
|
108
|
+
===========
|
|
109
|
+
|
|
110
|
+
https://gitter.im/dask-ms/community
|
|
111
|
+
|
|
112
|
+
Example Usage
|
|
113
|
+
=============
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
.. code-block:: python
|
|
117
|
+
|
|
118
|
+
import dask.array as da
|
|
119
|
+
from daskms import xds_from_table, xds_to_table
|
|
120
|
+
|
|
121
|
+
# Create xarray datasets from Measurement Set "WSRT.MS"
|
|
122
|
+
ds = xds_from_table("WSRT.MS")
|
|
123
|
+
# Set the flag Variable on first Dataset to it's inverse
|
|
124
|
+
ds[0]['flag'] = (ds[0].flag.dims, da.logical_not(ds[0].flag))
|
|
125
|
+
# Write the flag column back to the Measurement Set
|
|
126
|
+
xds_to_table(ds, "WSRT.MS", "FLAG").compute()
|
|
127
|
+
|
|
128
|
+
print ds
|
|
129
|
+
|
|
130
|
+
[<xarray.Dataset>
|
|
131
|
+
Dimensions: (chan: 64, corr: 4, row: 6552, uvw: 3)
|
|
132
|
+
Coordinates:
|
|
133
|
+
ROWID (row) int32 dask.array<shape=(6552,), chunksize=(6552,)>
|
|
134
|
+
Dimensions without coordinates: chan, corr, row, uvw
|
|
135
|
+
Data variables:
|
|
136
|
+
IMAGING_WEIGHT (row, chan) float32 dask.array<shape=(6552, 64), chunksize=(6552, 64)>
|
|
137
|
+
ANTENNA1 (row) int32 dask.array<shape=(6552,), chunksize=(6552,)>
|
|
138
|
+
STATE_ID (row) int32 dask.array<shape=(6552,), chunksize=(6552,)>
|
|
139
|
+
EXPOSURE (row) float64 dask.array<shape=(6552,), chunksize=(6552,)>
|
|
140
|
+
MODEL_DATA (row, chan, corr) complex64 dask.array<shape=(6552, 64, 4), chunksize=(6552, 64, 4)>
|
|
141
|
+
FLAG_ROW (row) bool dask.array<shape=(6552,), chunksize=(6552,)>
|
|
142
|
+
CORRECTED_DATA (row, chan, corr) complex64 dask.array<shape=(6552, 64, 4), chunksize=(6552, 64, 4)>
|
|
143
|
+
PROCESSOR_ID (row) int32 dask.array<shape=(6552,), chunksize=(6552,)>
|
|
144
|
+
WEIGHT (row, corr) float32 dask.array<shape=(6552, 4), chunksize=(6552, 4)>
|
|
145
|
+
FLAG (row, chan, corr) bool dask.array<shape=(6552, 64, 4), chunksize=(6552, 64, 4)>
|
|
146
|
+
TIME (row) float64 dask.array<shape=(6552,), chunksize=(6552,)>
|
|
147
|
+
SIGMA (row, corr) float32 dask.array<shape=(6552, 4), chunksize=(6552, 4)>
|
|
148
|
+
SCAN_NUMBER (row) int32 dask.array<shape=(6552,), chunksize=(6552,)>
|
|
149
|
+
INTERVAL (row) float64 dask.array<shape=(6552,), chunksize=(6552,)>
|
|
150
|
+
OBSERVATION_ID (row) int32 dask.array<shape=(6552,), chunksize=(6552,)>
|
|
151
|
+
TIME_CENTROID (row) float64 dask.array<shape=(6552,), chunksize=(6552,)>
|
|
152
|
+
ARRAY_ID (row) int32 dask.array<shape=(6552,), chunksize=(6552,)>
|
|
153
|
+
ANTENNA2 (row) int32 dask.array<shape=(6552,), chunksize=(6552,)>
|
|
154
|
+
DATA (row, chan, corr) complex64 dask.array<shape=(6552, 64, 4), chunksize=(6552, 64, 4)>
|
|
155
|
+
FEED1 (row) int32 dask.array<shape=(6552,), chunksize=(6552,)>
|
|
156
|
+
FEED2 (row) int32 dask.array<shape=(6552,), chunksize=(6552,)>
|
|
157
|
+
UVW (row, uvw) float64 dask.array<shape=(6552, 3), chunksize=(6552, 3)>
|
|
158
|
+
Attributes:
|
|
159
|
+
FIELD_ID: 0
|
|
160
|
+
DATA_DESC_ID: 0]
|
|
161
|
+
|
|
162
|
+
-----------
|
|
163
|
+
Limitations
|
|
164
|
+
-----------
|
|
165
|
+
|
|
166
|
+
1. Many Measurement Sets columns are defined as variably shaped,
|
|
167
|
+
but the actual data is fixed.
|
|
168
|
+
dask-ms_ will infer the shape of the
|
|
169
|
+
data from the first row and must be consistent
|
|
170
|
+
with that of other rows.
|
|
171
|
+
For example, this may be issue where multiple Spectral Windows
|
|
172
|
+
are present in the Measurement Set with differing channels
|
|
173
|
+
per SPW.
|
|
174
|
+
|
|
175
|
+
dask-ms_ works around this by partitioning the
|
|
176
|
+
Measurement Set into multiple datasets.
|
|
177
|
+
The first row's shape is used to infer the shape of the partition.
|
|
178
|
+
Thus, in the case of multiple Spectral Window's, we can partition
|
|
179
|
+
the Measurement Set by DATA_DESC_ID to create a dataset for
|
|
180
|
+
each Spectral Window.
|
|
181
|
+
|
|
182
|
+
.. _dask: https://dask.pydata.org
|
|
183
|
+
.. _dask-ms: https://github.com/ska-sa/dask-ms
|
|
184
|
+
.. _xarray: https://xarray.pydata.org
|
|
185
|
+
.. _python-casacore: https://github.com/casacore/python-casacore
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
daskms/__init__.py,sha256=xfC17pFzxdJWfowoYYuVCTFuVfK8jHgV2yIDnmCYDdo,724
|
|
2
|
+
daskms/array_api_utils.py,sha256=-y32NWrNoz1tVoXE4GSO8WJofLhj6kYMpm7J05d6cRg,2293
|
|
3
|
+
daskms/casacure_backend.py,sha256=MniTvgLSvwIKvSsp6KcjglC3WHPhu0U9sIaETgP9C1o,1729
|
|
4
|
+
daskms/columns.py,sha256=zyJ_NqAuF_0hgqOr4WlUJNgz8QqazMi6ZqMYniBCd2M,8813
|
|
5
|
+
daskms/config.py,sha256=qg3Bae1tErow4qcMZxbbtaxhToB4796js9JYjszSFsI,54
|
|
6
|
+
daskms/conftest.py,sha256=DPd6TLoCnVVFRs1DJWtTXKdAORNftnsms82gxaTOa10,7678
|
|
7
|
+
daskms/constants.py,sha256=H3lK8sXvBARdbhf3M-Xqiw7E2qFo5JfBzSE9ZlKjgmQ,126
|
|
8
|
+
daskms/dask_ms.py,sha256=OcHa8LcGY-dx8RD_CsqOqsdEx49OERPAaf8vqgJ7g_E,14423
|
|
9
|
+
daskms/dataset.py,sha256=iAukRuncJVeXbhiKbAsO6X-5fFOrLtuZ5gBmJ1RVyl0,16518
|
|
10
|
+
daskms/dataset_schema.py,sha256=MoZI0nrQBEzOHwUybx24CtRd3Y519IyZEK2TG3QhruQ,9428
|
|
11
|
+
daskms/example_data.py,sha256=r5B2PGwOrk-Gsa5T1yyiUI0DTTxTKXQETeOdUARlsNg,4274
|
|
12
|
+
daskms/expressions.py,sha256=FMqpgk06zf7j3Im2ws9Ez7OV07sMovK7kfs9W0SyE_Q,2697
|
|
13
|
+
daskms/fsspec_store.py,sha256=56018s3qrcL-BAe8IHC5CuKje6QbNITa0HSNbtFG2QQ,5491
|
|
14
|
+
daskms/multiton.py,sha256=ktKt-n4jqH_yU5ZO4gYU_XwQPBWVCXnQv1x_1PTxWX4,5757
|
|
15
|
+
daskms/optimisation.py,sha256=yoQG1zZff3skb8DPUapyC819HYqfBR8v8YKrO1vYwY4,11316
|
|
16
|
+
daskms/ordering.py,sha256=OczxThxvlI_qT-aENFOldERpbp2KFVYA5yz2BcZuZYc,7045
|
|
17
|
+
daskms/patterns.py,sha256=b5bFE0KEr9GDSQSxltCdkSEznGuRKG6OS_axoNeAgKk,12692
|
|
18
|
+
daskms/query.py,sha256=A--yDbl7XBN5GnR689xELbVXEC3ZFFI-2IbwQMvE6Ac,714
|
|
19
|
+
daskms/reads.py,sha256=vfeO-O8hb7fn8g9ZGAfw1Jbt10DSQ4BjWnD-7ZenbtI,18297
|
|
20
|
+
daskms/table.py,sha256=NFHGTvQNsZerpSw-1ruOvi3AArBfOnwX9jL0yR0Aju0,785
|
|
21
|
+
daskms/table_executor.py,sha256=dKBxbR1dQibTGWgTaCEWkwSALtg1QBAP0bWIk9FEzA4,1573
|
|
22
|
+
daskms/table_proxy.py,sha256=XWWm_UJaBvmCtYcy9RONyDX9w4lF0eo2JcD045j0BmU,13508
|
|
23
|
+
daskms/table_schemas.py,sha256=j5e3yg6d1reCni0-zujQM4Zfbq5IdFaNl9KRg2jbSXM,4940
|
|
24
|
+
daskms/testing.py,sha256=FrKevwPuMOvko805PZesTOs96Y7A-7rttKP23GwM74s,628
|
|
25
|
+
daskms/utils.py,sha256=cBGOGeNBN3PJIWZx-03H66fTYWRZwwZUJDn4ta1FFqA,7413
|
|
26
|
+
daskms/writes.py,sha256=2X4ukJYnMFoTXQzt4haxQBIsAYZlaZLCdOlp8yMwAi0,25379
|
|
27
|
+
daskms/apps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
28
|
+
daskms/apps/application.py,sha256=aWJFjNF2s7ENn6D5I-lZpJBrpLPj3p7eZ79SmmrZ-1g,315
|
|
29
|
+
daskms/apps/conftest.py,sha256=Uxc4XqWAfSEk-RbJuajLhzG57ZP1bWw5JrPvc62WWUs,1974
|
|
30
|
+
daskms/apps/convert.py,sha256=lFVXwL4izduEVfk82rxmMHOTFpokgYqbjauPKIAnHPg,7477
|
|
31
|
+
daskms/apps/entrypoint.py,sha256=3RyjRnJVhOchifPfkUQ5Sw1quBnL1LDX0pcJke1RnY8,433
|
|
32
|
+
daskms/apps/formats.py,sha256=uJm5xjMHcxuJMg5VAnh7Los2R_Ao7U5VLUi8_V7KCXg,7559
|
|
33
|
+
daskms/apps/fragments.py,sha256=902tkYypa78ukS7Q2SOAVnUi0_pniWqhtea4f3ViGBY,1398
|
|
34
|
+
daskms/apps/katdal_import.py,sha256=a_MA2q8SvnXTYKDGLtfdk5btc6iIjd0-1_IT4ptjQKE,2088
|
|
35
|
+
daskms/apps/conf/logging.ini,sha256=xlHgZWAqRrLijbplp7DmIbWaHzM6lC_6qsPPM59_hs4,323
|
|
36
|
+
daskms/apps/tests/test_chunk_transformer.py,sha256=5-PIXlb-oaJWp4PCW5XOXvO8Ya0FN4XxLpyQvlB2Xyw,292
|
|
37
|
+
daskms/apps/tests/test_convert.py,sha256=95Ow4P4XFPBXJFKfRuv4SMeSo9p3uRlDXwBOBJqQQNk,1681
|
|
38
|
+
daskms/descriptors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
39
|
+
daskms/descriptors/builder.py,sha256=B1RZ9bxyaQgBMai6uegGMpRENE08yqrLDta4hh46eWs,5498
|
|
40
|
+
daskms/descriptors/builder_factory.py,sha256=iVAaA39bcJUZ7-18PIt5ZtQpa7KZQVsYrIxSnKRCTxc,4153
|
|
41
|
+
daskms/descriptors/ms.py,sha256=8OjNKlaHA60OH5UixCRbDNnyIiCGEifwqVAnPhlW5u8,10027
|
|
42
|
+
daskms/descriptors/ms_subtable.py,sha256=kJbHVkpL3D9z_yjKFMyUcsWx71LKvXKEcoA9QYAYTCQ,1374
|
|
43
|
+
daskms/descriptors/ratt_ms.py,sha256=gYJA_UPXBCM-AE2U2HIX1IJeGsg4-dqX5kM75oBKt-Y,641
|
|
44
|
+
daskms/descriptors/register_default_builders.py,sha256=NP981LWM_OV3eSc5x9zHJEJ6fTH0J-eInJuJQNp1ysQ,150
|
|
45
|
+
daskms/descriptors/tests/test_builder_factory.py,sha256=6k71LZkt7OiwEsdAcXg0j6TGX0TRaQDpbJvaa43idMg,2224
|
|
46
|
+
daskms/descriptors/tests/test_default_builder.py,sha256=cyFEQvYXGZlevykPKRaOVYPFlbnrNnYbM7COyeF6l3Y,3058
|
|
47
|
+
daskms/descriptors/tests/test_ms_builder.py,sha256=3-5X9_2qptAMFcNoJO9H7DEyuBc-o1e0W0MsWN7dbdg,3651
|
|
48
|
+
daskms/descriptors/tests/test_ms_subtable_builder.py,sha256=DVsHDZ_lQAKCgTAKyprz3V_h2laKn8jErA2syWrp2_Y,1205
|
|
49
|
+
daskms/experimental/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
50
|
+
daskms/experimental/utils.py,sha256=6y80hhtbu6smUXpzmYcCoqzLJv2vvl2BT5rLcz9LoEk,6723
|
|
51
|
+
daskms/experimental/arrow/__init__.py,sha256=JvAAR-VCY5q3DlUzELWhHVigVl353K7POcJ4ROEuYtw,149
|
|
52
|
+
daskms/experimental/arrow/arrow_schema.py,sha256=vTIg5RxImIBdHPMwLRRzfL_hfylFfXSw5Jnam9Smiyk,5567
|
|
53
|
+
daskms/experimental/arrow/extension_types.py,sha256=pJzBJi_d5BEZY9WRN0ZGNJuRv8G0I96Ghemi8y2FrT8,7041
|
|
54
|
+
daskms/experimental/arrow/reads.py,sha256=1gfbxnptU9DyfoNZ_uxiSfgN2-4VPI_3nofQhsmOn0g,11260
|
|
55
|
+
daskms/experimental/arrow/require_arrow.py,sha256=9C334unVi7DJ_lngaSut9goCzAnTmpl-OEftjJZJqUk,149
|
|
56
|
+
daskms/experimental/arrow/writes.py,sha256=iE1moCFjk-MpbvxXA4cCKCngI5KFIryR546Mn1yc6sY,5851
|
|
57
|
+
daskms/experimental/arrow/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
58
|
+
daskms/experimental/arrow/tests/test_extensions.py,sha256=u0WgOEQAdZwCWa2O8zoX1fw3K-Y9Et-tKN1PUP26mBU,1920
|
|
59
|
+
daskms/experimental/arrow/tests/test_parquet.py,sha256=OxDUWi99eP_060mo9CSaitkWgYfbLgm-kt-Zk78xVUI,6879
|
|
60
|
+
daskms/experimental/fragments/__init__.py,sha256=wmQIcswWIksJF4cpnh5WIGtF5Cb59MuYBUoFAF2O3KY,8624
|
|
61
|
+
daskms/experimental/fragments/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
62
|
+
daskms/experimental/fragments/tests/test_fragments.py,sha256=LjxmAyt_BJaD1NkxWFAVTAJaLWzd4Z1QQZGtO6aEnY8,9720
|
|
63
|
+
daskms/experimental/katdal/__init__.py,sha256=Qdyg3Ytsu8K-g7C0TRwgTQlKMxbuJsSPvf0umFQXR8Q,84
|
|
64
|
+
daskms/experimental/katdal/constants.py,sha256=_TcD1pdzR5QD_CGjVdAPKkpvD45lu7F_ObSKuA7o_KQ,519
|
|
65
|
+
daskms/experimental/katdal/corr_products.py,sha256=MsgVKzj8u8oMiAokhDBJ_fF9g9Nu3IcEMiUdmZpo_mo,2779
|
|
66
|
+
daskms/experimental/katdal/katdal_import.py,sha256=WRApXIgn5T2Xm-7VfTx_hBrVffwrZx6ydOb7eMRP9KY,4447
|
|
67
|
+
daskms/experimental/katdal/meerkat_antennas.py,sha256=5at7v8MhSyeCVV2p4grgeT3rIXxSZEputpG4Au-aG-M,5107
|
|
68
|
+
daskms/experimental/katdal/mock_dataset.py,sha256=hvaATluVfXRv_rxiaymTgnSEZZRa3Fq9z6VqBH5PdOU,2835
|
|
69
|
+
daskms/experimental/katdal/msv2_facade.py,sha256=64gq0TBInHfJFojq6INhgZyVRpK9DdqUTEBJj-lU324,25598
|
|
70
|
+
daskms/experimental/katdal/transpose.py,sha256=fdDnjJ_anREIQLo42vM0ww0iblzVs7zYOVocsDc9sgw,4291
|
|
71
|
+
daskms/experimental/katdal/uvw.py,sha256=OOpD2IK4niQ37a2zLiSF_UwBB1QOoHFkcDO3UXCJwnQ,2580
|
|
72
|
+
daskms/experimental/katdal/tests/conftest.py,sha256=0WV07tDqNoI4v7yQ-k5SKmCctqrr1rPdE59ov6aOs0Q,2052
|
|
73
|
+
daskms/experimental/katdal/tests/test_chunkstore.py,sha256=FVCAVpyChGd-7__LIzzWcvYZ90RZzUCXyE0ZdbNFH_E,5955
|
|
74
|
+
daskms/experimental/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
75
|
+
daskms/experimental/tests/test_utils.py,sha256=gmIj810wLQxD0ZBXF9PBZYoNf_0smvwFlJ53_2B1Wl4,3764
|
|
76
|
+
daskms/experimental/zarr/__init__.py,sha256=yAsDb76EIRGG06YgOmaqPoWWhVWIBYo9OZcCFk5X5tU,17712
|
|
77
|
+
daskms/experimental/zarr/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
78
|
+
daskms/experimental/zarr/tests/conftest.py,sha256=Sh-7ni3leMyMVKgMdGfQAjoqx8uP4zfGWlu8u-NHNsY,1731
|
|
79
|
+
daskms/experimental/zarr/tests/test_zarr.py,sha256=nEuaqfB1hY4mnpdVVBb_hJfs9QAkue2Ezlo2Dv646Lo,14736
|
|
80
|
+
daskms/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
81
|
+
daskms/tests/test_array_api_utils.py,sha256=j8-679LtFC65XTIVphx3QcZYSHuwaC1nrxpn1mnEDwY,9104
|
|
82
|
+
daskms/tests/test_columns.py,sha256=y4MCsAVQ2yb1zKroWEXVaSt0m27R973yd5-HkHFPW_Q,2024
|
|
83
|
+
daskms/tests/test_dataset.py,sha256=jChg3h_hN0UtJfENl-VQ5fWwiazpN7fAkHV7S7CVRyA,20669
|
|
84
|
+
daskms/tests/test_dataset_keywords.py,sha256=HrWjv6EOhnCWXLbbHYPMByV6vzHXRj6gDnZp4pCM5_E,3335
|
|
85
|
+
daskms/tests/test_dataset_schema.py,sha256=qV09DmTm_yc3DWUHN3GM01aUusB3dNzRDZJgUlUTb7E,1636
|
|
86
|
+
daskms/tests/test_executor.py,sha256=LBXYilV_2e0tl0fOONwLE4JgZzyUr3j5RgsImf7vvug,1755
|
|
87
|
+
daskms/tests/test_expression.py,sha256=Q9_A_C2Qq4ZE6IuVdvmlz42VvQsIIaXMa1AVpnwo8F0,789
|
|
88
|
+
daskms/tests/test_fsspec_store.py,sha256=UDlbXVtl6WKJ62pFX9S_mKdv0_KNfTrpuBkFfF35sU4,4739
|
|
89
|
+
daskms/tests/test_github.py,sha256=tEGKuERY-HI_A4J450dKcL5VH8FEeA9fuaExfyNqZT8,801
|
|
90
|
+
daskms/tests/test_ms_creation.py,sha256=E2QOOxEbj9BrL9p-yL7Sv7ZN5_MC20giAWe7MABsjYE,12133
|
|
91
|
+
daskms/tests/test_ms_read_and_update.py,sha256=oNtyDoz1Do1rcH-qsgZHzJv_1ynP9IBXrF_06bfU0IM,23369
|
|
92
|
+
daskms/tests/test_multiton_metaclass.py,sha256=ZO4CBYLobMBVgTW4BzaIH5y-0nLW5CjNC3PytuyyqBU,4035
|
|
93
|
+
daskms/tests/test_optimisation.py,sha256=uQULy41TYh6bjsHp7EkyA6wAE_9No-IPiA4CUtVIov4,4500
|
|
94
|
+
daskms/tests/test_optional.py,sha256=fXIlTcJ8XqE1GKmkRlf1WS7i6E_Iu3BrdX-2_Kv9x_E,12555
|
|
95
|
+
daskms/tests/test_ordering.py,sha256=oODT0SvrJGFslh6SgSxnqxfAtgmQsv49mp7yU_ie3mc,7041
|
|
96
|
+
daskms/tests/test_patterns.py,sha256=7nK08icopiqWZ9zzdzAuYWUKo20bAcb-LQnTZE0bNN4,3290
|
|
97
|
+
daskms/tests/test_storage.py,sha256=VMgOFB5XOnGPvYREnMAJOT1boRfATrSpU-3TjotqHJE,1798
|
|
98
|
+
daskms/tests/test_stress.py,sha256=EYrYK07PIpTYr8YCDmA1xGFty6R3h2foFBzWL26kdrU,813
|
|
99
|
+
daskms/tests/test_table.py,sha256=Ns-gvvoPYaTKsx2JPA30XEV81N08btc4clfxYykpn4U,479
|
|
100
|
+
daskms/tests/test_table_proxy.py,sha256=DG4c3rs-dQoF9EhWBPdG621yToKQ4piQMY_zdxZPpNA,6727
|
|
101
|
+
daskms/tests/test_table_schemas.py,sha256=H_vQuQAnlDQQoqDJ0hJ3VDFMg_12a9Cjtge_fSMkfqY,1120
|
|
102
|
+
daskms/tests/test_utils.py,sha256=qBnxFdsiwiAnBY7tZqaEJba8ibPmHREfk_VAgwv8tNo,3037
|
|
103
|
+
dask_ms_2-0.2.33.dist-info/METADATA,sha256=BuGQ2FeqV2ChcK6LYpA1LVaJXgx94K9lAKHIyDk8rTI,7342
|
|
104
|
+
dask_ms_2-0.2.33.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
105
|
+
dask_ms_2-0.2.33.dist-info/entry_points.txt,sha256=k__sXeytbCqZKE703QSwM4u5OGLItzfItMNV_Eh_zmM,95
|
|
106
|
+
dask_ms_2-0.2.33.dist-info/licenses/AUTHORS.rst,sha256=pJcOHm3n3juZT_YFItL8EMjLavae5YVVr92BkMJE228,157
|
|
107
|
+
dask_ms_2-0.2.33.dist-info/licenses/LICENSE,sha256=GJNXP2pkEOswvJWWzkMDLxPWqCu_hn-IkRfBdq-26Ic,1565
|
|
108
|
+
dask_ms_2-0.2.33.dist-info/RECORD,,
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
Copyright (c) 2019 - 2020, National Research Foundation
|
|
2
|
+
(South African Radio Astronomy Observatory)
|
|
3
|
+
All rights reserved.
|
|
4
|
+
|
|
5
|
+
Redistribution and use in source and binary forms, with or without modification,
|
|
6
|
+
are permitted provided that the following conditions are met:
|
|
7
|
+
|
|
8
|
+
1. Redistributions of source code must retain the above copyright notice,
|
|
9
|
+
this list of conditions and the following disclaimer.
|
|
10
|
+
|
|
11
|
+
2. Redistributions in binary form must reproduce the above copyright notice,
|
|
12
|
+
this list of conditions and the following disclaimer in the documentation
|
|
13
|
+
and/or other materials provided with the distribution.
|
|
14
|
+
|
|
15
|
+
3. Neither the name of the copyright holder nor the names of its contributors
|
|
16
|
+
may be used to endorse or promote products derived from this software
|
|
17
|
+
without specific prior written permission.
|
|
18
|
+
|
|
19
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
20
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
|
|
21
|
+
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
22
|
+
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
|
23
|
+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
24
|
+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
|
25
|
+
OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
26
|
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
27
|
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
28
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
daskms/__init__.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
|
|
3
|
+
# Activate the casacure backend (DASK_MS_BACKEND=casacure) before any
|
|
4
|
+
# module pulls in casacore.tables via lazy_import.
|
|
5
|
+
from daskms.casacure_backend import activate_casacure_backend
|
|
6
|
+
|
|
7
|
+
import logging
|
|
8
|
+
|
|
9
|
+
activate_casacure_backend()
|
|
10
|
+
|
|
11
|
+
__author__ = """Simon Perkins"""
|
|
12
|
+
__email__ = "sperkins@ska.ac.za"
|
|
13
|
+
__version__ = "0.2.33"
|
|
14
|
+
|
|
15
|
+
logging.getLogger(__name__).addHandler(logging.NullHandler())
|
|
16
|
+
|
|
17
|
+
from daskms.dask_ms import (
|
|
18
|
+
xds_from_table, # noqa
|
|
19
|
+
xds_from_ms, # noqa
|
|
20
|
+
xds_from_storage_ms, # noqa
|
|
21
|
+
xds_from_storage_table, # noqa
|
|
22
|
+
xds_to_table, # noqa
|
|
23
|
+
xds_to_storage_table,
|
|
24
|
+
) # noqa
|
|
25
|
+
|
|
26
|
+
from daskms.dataset import Dataset, Variable # noqa
|
|
27
|
+
from daskms.table_proxy import TableProxy # noqa
|
daskms/apps/__init__.py
ADDED
|
File without changes
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import abc
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class Application(abc.ABC):
|
|
5
|
+
@abc.abstractmethod
|
|
6
|
+
def __init__(self, args, log):
|
|
7
|
+
raise NotImplementedError
|
|
8
|
+
|
|
9
|
+
@abc.abstractclassmethod
|
|
10
|
+
def setup_parser(cls, parser):
|
|
11
|
+
raise NotImplementedError
|
|
12
|
+
|
|
13
|
+
@abc.abstractmethod
|
|
14
|
+
def execute(self):
|
|
15
|
+
raise NotImplementedError
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
[loggers]
|
|
2
|
+
keys=root
|
|
3
|
+
|
|
4
|
+
[handlers]
|
|
5
|
+
keys=consoleHandler
|
|
6
|
+
|
|
7
|
+
[formatters]
|
|
8
|
+
keys=extend
|
|
9
|
+
|
|
10
|
+
[logger_root]
|
|
11
|
+
level=INFO
|
|
12
|
+
handlers=consoleHandler
|
|
13
|
+
propagate=0
|
|
14
|
+
|
|
15
|
+
[handler_consoleHandler]
|
|
16
|
+
class=StreamHandler
|
|
17
|
+
level=INFO
|
|
18
|
+
formatter=extend
|
|
19
|
+
args = ()
|
|
20
|
+
# args=(sys.stdout)
|
|
21
|
+
|
|
22
|
+
[formatter_extend]
|
|
23
|
+
format=%(asctime)s - dask-ms - %(levelname)s - %(message)s
|
daskms/apps/conftest.py
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
from appdirs import user_cache_dir
|
|
2
|
+
from hashlib import sha256
|
|
3
|
+
import logging
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
import tarfile
|
|
6
|
+
|
|
7
|
+
import pytest
|
|
8
|
+
|
|
9
|
+
log = logging.getLogger(__file__)
|
|
10
|
+
|
|
11
|
+
TAU_MS = "HLTau_B6cont.calavg.tav300s"
|
|
12
|
+
TAU_MS_TAR = f"{TAU_MS}.tar.xz"
|
|
13
|
+
TAU_MS_TAR_HASH = "fc2ce9261817dfd88bbdd244c8e9e58ae0362173938df6ef2a587b1823147f70"
|
|
14
|
+
DATA_URL = f"s3://ratt-public-data/test-data/{TAU_MS_TAR}"
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def download_tau_ms(tau_ms_tar):
|
|
18
|
+
if tau_ms_tar.exists():
|
|
19
|
+
with open(tau_ms_tar, "rb") as f:
|
|
20
|
+
digest = sha256()
|
|
21
|
+
|
|
22
|
+
while data := f.read(2**20):
|
|
23
|
+
digest.update(data)
|
|
24
|
+
|
|
25
|
+
if digest.hexdigest() == TAU_MS_TAR_HASH:
|
|
26
|
+
return
|
|
27
|
+
|
|
28
|
+
tau_ms_tar.unlink(missing_ok=True)
|
|
29
|
+
raise ValueError(
|
|
30
|
+
f"sha256 digest '{digest.hexdigest()}' "
|
|
31
|
+
f"of {tau_ms_tar} does not match "
|
|
32
|
+
f"{TAU_MS_TAR_HASH}"
|
|
33
|
+
)
|
|
34
|
+
else:
|
|
35
|
+
s3fs = pytest.importorskip("s3fs")
|
|
36
|
+
s3 = s3fs.S3FileSystem(anon=True)
|
|
37
|
+
|
|
38
|
+
for attempt in range(3):
|
|
39
|
+
with s3.open(DATA_URL, "rb") as fin, open(tau_ms_tar, "wb") as fout:
|
|
40
|
+
digest = sha256()
|
|
41
|
+
|
|
42
|
+
while data := fin.read(2**20):
|
|
43
|
+
digest.update(data)
|
|
44
|
+
fout.write(data)
|
|
45
|
+
|
|
46
|
+
if digest.hexdigest() == TAU_MS_TAR_HASH:
|
|
47
|
+
return
|
|
48
|
+
|
|
49
|
+
log.warning("Download of %s failed on attempt %d", DATA_URL, attempt)
|
|
50
|
+
tau_ms_tar.unlink(missing_ok=True)
|
|
51
|
+
|
|
52
|
+
raise ValueError(f"Download of {DATA_URL} failed {attempt} times")
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@pytest.fixture(scope="function")
|
|
56
|
+
def tau_ms(tmp_path_factory):
|
|
57
|
+
cache_dir = Path(user_cache_dir("dask-ms")) / "test-data"
|
|
58
|
+
cache_dir.mkdir(parents=True, exist_ok=True)
|
|
59
|
+
tau_ms_tar = cache_dir / TAU_MS_TAR
|
|
60
|
+
|
|
61
|
+
download_tau_ms(tau_ms_tar)
|
|
62
|
+
msdir = tmp_path_factory.mktemp("taums")
|
|
63
|
+
|
|
64
|
+
with tarfile.open(tau_ms_tar) as tar:
|
|
65
|
+
tar.extractall(msdir)
|
|
66
|
+
|
|
67
|
+
yield msdir / TAU_MS
|