xarray-sql 0.1.0__tar.gz → 0.1.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/CONTRIBUTING.md +1 -2
  2. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/PKG-INFO +70 -23
  3. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/README.md +69 -22
  4. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/xarray_sql/df.py +68 -9
  5. xarray_sql-0.1.2/xarray_sql/df_test.py +387 -0
  6. xarray_sql-0.1.2/xarray_sql/sql_test.py +813 -0
  7. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/xarray_sql.egg-info/PKG-INFO +70 -23
  8. xarray_sql-0.1.0/xarray_sql/df_test.py +0 -182
  9. xarray_sql-0.1.0/xarray_sql/sql_test.py +0 -64
  10. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/.github/workflows/ci.yml +0 -0
  11. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/.github/workflows/lint.yml +0 -0
  12. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/.github/workflows/publish.yml +0 -0
  13. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/.gitignore +0 -0
  14. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/.python-version +0 -0
  15. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/LICENSE +0 -0
  16. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/perf_tests/README.md +0 -0
  17. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/perf_tests/compute_air.py +0 -0
  18. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/perf_tests/compute_air.py-2024-02-26T22:57:59+07:00.svg +0 -0
  19. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/perf_tests/compute_air.py-2024-03-03T07:42:36+05:30.svg +0 -0
  20. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/perf_tests/compute_air.py-2024-03-03T07:50:18+05:30.svg +0 -0
  21. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/perf_tests/groupby_air.py +0 -0
  22. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/perf_tests/groupby_air.py-2024-02-15T13:01:22+07:00.svg +0 -0
  23. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/perf_tests/groupby_air.py-2024-02-15T15:03:53+07:00.svg +0 -0
  24. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/perf_tests/groupby_air.py-2024-02-17T18:42:14+07:00.svg +0 -0
  25. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/perf_tests/groupby_air.py-2024-02-26T23:04:34+07:00.svg +0 -0
  26. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/perf_tests/groupby_air.py-2024-03-03T07:43:22+05:30.svg +0 -0
  27. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/perf_tests/groupby_air.py-2024-03-03T07:50:06+05:30.svg +0 -0
  28. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/perf_tests/groupby_air_full.py +0 -0
  29. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/perf_tests/groupby_air_full.py-2024-02-26T23:06:38+07:00.svg +0 -0
  30. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/perf_tests/groupby_air_full.py-2024-03-03T07:44:17+05:30.svg +0 -0
  31. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/perf_tests/groupby_air_full.py-2024-03-03T07:49:53+05:30.svg +0 -0
  32. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/perf_tests/open_era5.py +0 -0
  33. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/perf_tests/open_era5.py-2024-02-18T17:21:44+07:00.svg +0 -0
  34. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/perf_tests/open_era5.py-2024-02-18T17:33:02+07:00.svg +0 -0
  35. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/perf_tests/open_era5.py-2024-02-19T17:52:51+07:00.svg +0 -0
  36. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/perf_tests/profile.sh +0 -0
  37. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/perf_tests/sanity.py +0 -0
  38. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/perf_tests/sanity.py-2024-02-15T09:08:28+07:00.svg +0 -0
  39. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/perf_tests/sanity.py-2024-02-15T09:15:07+07:00.svg +0 -0
  40. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/perf_tests/sanity.py-2024-02-15T12:51:37+07:00.svg +0 -0
  41. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/perf_tests/sanity.py-2024-02-15T15:04:48+07:00.svg +0 -0
  42. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/perf_tests/sanity.py-2024-02-17T18:42:08+07:00.svg +0 -0
  43. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/perf_tests/sanity.py-2024-02-26T23:00:00+07:00.svg +0 -0
  44. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/perf_tests/sanity.py-2024-03-03T07:45:47+05:30.svg +0 -0
  45. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/perf_tests/sanity.py-2024-03-03T07:49:44+05:30.svg +0 -0
  46. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/pyproject.toml +0 -0
  47. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/setup.cfg +0 -0
  48. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/uv.lock +0 -0
  49. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/xarray_sql/__init__.py +0 -0
  50. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/xarray_sql/core.py +0 -0
  51. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/xarray_sql/df_integrationtest.py +0 -0
  52. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/xarray_sql/sql.py +0 -0
  53. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/xarray_sql.egg-info/SOURCES.txt +0 -0
  54. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/xarray_sql.egg-info/dependency_links.txt +0 -0
  55. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/xarray_sql.egg-info/requires.txt +0 -0
  56. {xarray_sql-0.1.0 → xarray_sql-0.1.2}/xarray_sql.egg-info/top_level.txt +0 -0
@@ -12,8 +12,7 @@ reading [Xarray's contributing guide](https://docs.xarray.dev/en/stable/contribu
12
12
 
13
13
  ## Developer setup
14
14
 
15
- 0. Create a project-specific python
16
- environment with `uv`: https://docs.astral.sh/uv/getting-started/installation/
15
+ 0. We use `uv` to manage the project: https://docs.astral.sh/uv/getting-started/installation/
17
16
  1. Clone the repository (bonus: [via SSH](https://docs.github.com/en/authentication/connecting-to-github-with-ssh/adding-a-new-ssh-key-to-your-github-account))
18
17
  and `cd xarray_sql` (the project root).
19
18
  2. Install dev dependencies via: `uv sync --dev`
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: xarray_sql
3
- Version: 0.1.0
3
+ Version: 0.1.2
4
4
  Summary: Querry Xarray with SQL.
5
5
  Author-email: Alexander Merose <al@merose.com>
6
6
  License: Apache-2.0
@@ -46,19 +46,35 @@ pip install xarray-sql
46
46
 
47
47
  ## What is this?
48
48
 
49
- This is an experiment to provide a SQL interface for raster data.
49
+ This is an experiment to provide a SQL interface for array datasets.
50
50
 
51
51
  ```python
52
52
  import xarray as xr
53
- import xarray_sql as qr
53
+ import xarray_sql as xql
54
54
 
55
55
  ds = xr.tutorial.open_dataset('air_temperature')
56
56
 
57
57
  # The same as a dask-sql Context; i.e. an Apache DataFusion Context.
58
- c = qr.XarrayContext()
59
- c.from_dataset('air', ds, chunks=dict(time=24))
60
-
61
- df = c.sql('''
58
+ ctx = xql.XarrayContext()
59
+ ctx.from_dataset('air', ds, chunks=dict(time=24)) # the dataset needs to be chunked!
60
+ # DataFrame()
61
+ # +------+---------------------+-------+--------------------+
62
+ # | lat | time | lon | air |
63
+ # +------+---------------------+-------+--------------------+
64
+ # | 75.0 | 2013-01-01T00:00:00 | 200.0 | 241.20000000000002 |
65
+ # | 75.0 | 2013-01-01T00:00:00 | 202.5 | 242.5 |
66
+ # | 75.0 | 2013-01-01T00:00:00 | 205.0 | 243.5 |
67
+ # | 75.0 | 2013-01-01T00:00:00 | 207.5 | 244.0 |
68
+ # | 75.0 | 2013-01-01T00:00:00 | 210.0 | 244.1 |
69
+ # | 75.0 | 2013-01-01T00:00:00 | 212.5 | 243.89000000000001 |
70
+ # | 75.0 | 2013-01-01T00:00:00 | 215.0 | 243.6 |
71
+ # | 75.0 | 2013-01-01T00:00:00 | 217.5 | 243.1 |
72
+ # | 75.0 | 2013-01-01T00:00:00 | 220.0 | 242.5 |
73
+ # | 75.0 | 2013-01-01T00:00:00 | 222.5 | 241.89000000000001 |
74
+ # +------+---------------------+-------+--------------------+
75
+ # Data truncated.
76
+
77
+ result = ctx.sql('''
62
78
  SELECT
63
79
  "lat", "lon", AVG("air") as air_total
64
80
  FROM
@@ -66,16 +82,36 @@ df = c.sql('''
66
82
  GROUP BY
67
83
  "lat", "lon"
68
84
  ''')
85
+ # DataFrame()
86
+ # +------+-------+--------------------+
87
+ # | lat | lon | air_total |
88
+ # +------+-------+--------------------+
89
+ # | 75.0 | 205.0 | 259.88662671232834 |
90
+ # | 75.0 | 207.5 | 259.48268150684896 |
91
+ # | 75.0 | 230.0 | 258.9192123287667 |
92
+ # | 75.0 | 275.0 | 257.07574315068456 |
93
+ # | 75.0 | 322.5 | 250.11792123287654 |
94
+ # | 75.0 | 325.0 | 250.81590068493125 |
95
+ # | 72.5 | 205.0 | 262.74933904109537 |
96
+ # | 72.5 | 207.5 | 262.5384315068488 |
97
+ # | 72.5 | 230.0 | 260.8287945205475 |
98
+ # | 72.5 | 275.0 | 257.30633219178037 |
99
+ # +------+-------+--------------------+
100
+ # Data truncated.
101
+ #
69
102
 
70
103
  # A table of the average temperature for each location across time.
71
- df.to_pandas()
72
-
73
- # Alternatively, you can just create the DataFrame from the Dataset:
74
- df = qr.read_xarray(ds).to_pandas()
104
+ df = result.to_pandas()
75
105
  df.head()
106
+ # lat lon air_total
107
+ # 0 75.0 210.0 259.016562
108
+ # 1 75.0 222.5 258.362212
109
+ # 2 75.0 237.5 258.318240
110
+ # 3 75.0 267.5 256.928497
111
+ # 4 75.0 285.0 261.614103
76
112
  ```
77
113
 
78
- Succinctly, we "pivot" Xarray Datasets to treat them like tables so we can run
114
+ Succinctly, we "pivot" Xarray Datasets (with consistent dimensions) to treat them like tables so we can run
79
115
  SQL queries against them.
80
116
 
81
117
  ## Why build this?
@@ -93,18 +129,18 @@ A few reasons:
93
129
 
94
130
  This is a light-weight way to prove the value of the interface.
95
131
 
96
- The larger goal is to explore the hypothesis that [Pangeo](https://pangeo.io/)
97
- is a scientific database. Here, xarray-sql can be thought of as a missing DB
98
- front end.
132
+ The larger goal is to explore the hypothesis that the [Pangeo](https://pangeo.io/)
133
+ ecosystem is a scientific database. Here, xarray-sql can be thought of as a missing
134
+ DB front end.
99
135
 
100
136
  ## How does it work?
101
137
 
102
- All chunks in an Xarray Dataset are transformed into a Dask DataFrame via
138
+ All chunks in a Xarray Dataset are transformed into a Dask DataFrame via
103
139
  `from_map()` and `to_dataframe()`. For SQL support, we just use `dask-sql`.
104
140
  That's it!
105
141
 
106
- _2025 update_: This library now implements a dask-like `from_map` interface in
107
- pure `datafusion` and `pyarrow`, but works with the same principle!
142
+ _2025 update_: This library now implements a Dask-like `from_map` interface in
143
+ pure DataFusion and PyArrow, but works with the same principle!
108
144
 
109
145
  ## Why does this work?
110
146
 
@@ -112,14 +148,15 @@ Underneath Xarray, Dask, and Pandas, there are NumPy arrays. These are paged in
112
148
  chunks and represented contiguously in memory. It is only a matter of metadata
113
149
  that breaks them up into ndarrays. `to_dataframe()`
114
150
  just changes this metadata (via a `ravel()`/`reshape()`), back into a column
115
- amenable to a DataFrame.
116
-
117
- There is added overhead from duplicating dimensions as columns, which we see as
118
- worth the convenience of DataFrames.
151
+ amenable to a DataFrame. We take advantage of this light weight metadata change to
152
+ make chunked information scannable by a DB engine (DataFusion).
119
153
 
120
154
  ## What are the current limitations?
121
155
 
122
- _2025 update_: TBD, `datafusion` provides a whole new world!
156
+ _2025 update_: TBD, DataFusion provides a whole new world! Currently, we're looking for
157
+ early users – "tire kickers", if you will. We'd love your input to shape the direction of this
158
+ project! Please, give this a try and [file issues](https://github.com/alxmrs/xarray-sql/issues) as
159
+ you see fit. Check out our [contributing guide](CONTRIBUTING.md), too 😉.
123
160
 
124
161
  ## What would a deeper integration look like?
125
162
 
@@ -134,6 +171,13 @@ parquet would open up integrations to numerous tools like dask, pyarrow, duckdb,
134
171
  and BigQuery. More thoughts on this
135
172
  in [#4](https://github.com/alxmrs/xarray-sql/issues/4).
136
173
 
174
+ _2025 update_: Something like this is being built across a few projects! The ones I know about are:
175
+ - [CartoDB's Raquet](https://github.com/CartoDB/raquet)
176
+ - The DataFusion community's [arrow-zarr](https://github.com/datafusion-contrib/arrow-zarr)
177
+
178
+ As of writing, this project is [amid integrating](https://github.com/alxmrs/xarray-sql/pull/69) a
179
+ rust-based DataFusion backend provided by arrow-zarr.
180
+
137
181
  ## Sponsors & Contributors
138
182
 
139
183
  I want to give a special thanks to the following folks and institutions:
@@ -143,6 +187,9 @@ I want to give a special thanks to the following folks and institutions:
143
187
  - Jake Wall and AI2/Ecoscope for compute resources and key use cases.
144
188
  - Charles Stern, Stephan Hoyer, Alexander Kmoch, Wei Ji, and Qiusheng Wu
145
189
  for the early review and discussion of this project.
190
+ - Tom Nichols, Kyle Barron, Tom White, and Maxime Dion for the [Array Working
191
+ Group](https://discourse.pangeo.io/t/new-working-group-for-distributed-array-computing/2734)
192
+ and DataFusion-specific collaboration.
146
193
 
147
194
  ## License
148
195
 
@@ -11,19 +11,35 @@ pip install xarray-sql
11
11
 
12
12
  ## What is this?
13
13
 
14
- This is an experiment to provide a SQL interface for raster data.
14
+ This is an experiment to provide a SQL interface for array datasets.
15
15
 
16
16
  ```python
17
17
  import xarray as xr
18
- import xarray_sql as qr
18
+ import xarray_sql as xql
19
19
 
20
20
  ds = xr.tutorial.open_dataset('air_temperature')
21
21
 
22
22
  # The same as a dask-sql Context; i.e. an Apache DataFusion Context.
23
- c = qr.XarrayContext()
24
- c.from_dataset('air', ds, chunks=dict(time=24))
25
-
26
- df = c.sql('''
23
+ ctx = xql.XarrayContext()
24
+ ctx.from_dataset('air', ds, chunks=dict(time=24)) # the dataset needs to be chunked!
25
+ # DataFrame()
26
+ # +------+---------------------+-------+--------------------+
27
+ # | lat | time | lon | air |
28
+ # +------+---------------------+-------+--------------------+
29
+ # | 75.0 | 2013-01-01T00:00:00 | 200.0 | 241.20000000000002 |
30
+ # | 75.0 | 2013-01-01T00:00:00 | 202.5 | 242.5 |
31
+ # | 75.0 | 2013-01-01T00:00:00 | 205.0 | 243.5 |
32
+ # | 75.0 | 2013-01-01T00:00:00 | 207.5 | 244.0 |
33
+ # | 75.0 | 2013-01-01T00:00:00 | 210.0 | 244.1 |
34
+ # | 75.0 | 2013-01-01T00:00:00 | 212.5 | 243.89000000000001 |
35
+ # | 75.0 | 2013-01-01T00:00:00 | 215.0 | 243.6 |
36
+ # | 75.0 | 2013-01-01T00:00:00 | 217.5 | 243.1 |
37
+ # | 75.0 | 2013-01-01T00:00:00 | 220.0 | 242.5 |
38
+ # | 75.0 | 2013-01-01T00:00:00 | 222.5 | 241.89000000000001 |
39
+ # +------+---------------------+-------+--------------------+
40
+ # Data truncated.
41
+
42
+ result = ctx.sql('''
27
43
  SELECT
28
44
  "lat", "lon", AVG("air") as air_total
29
45
  FROM
@@ -31,16 +47,36 @@ df = c.sql('''
31
47
  GROUP BY
32
48
  "lat", "lon"
33
49
  ''')
50
+ # DataFrame()
51
+ # +------+-------+--------------------+
52
+ # | lat | lon | air_total |
53
+ # +------+-------+--------------------+
54
+ # | 75.0 | 205.0 | 259.88662671232834 |
55
+ # | 75.0 | 207.5 | 259.48268150684896 |
56
+ # | 75.0 | 230.0 | 258.9192123287667 |
57
+ # | 75.0 | 275.0 | 257.07574315068456 |
58
+ # | 75.0 | 322.5 | 250.11792123287654 |
59
+ # | 75.0 | 325.0 | 250.81590068493125 |
60
+ # | 72.5 | 205.0 | 262.74933904109537 |
61
+ # | 72.5 | 207.5 | 262.5384315068488 |
62
+ # | 72.5 | 230.0 | 260.8287945205475 |
63
+ # | 72.5 | 275.0 | 257.30633219178037 |
64
+ # +------+-------+--------------------+
65
+ # Data truncated.
66
+ #
34
67
 
35
68
  # A table of the average temperature for each location across time.
36
- df.to_pandas()
37
-
38
- # Alternatively, you can just create the DataFrame from the Dataset:
39
- df = qr.read_xarray(ds).to_pandas()
69
+ df = result.to_pandas()
40
70
  df.head()
71
+ # lat lon air_total
72
+ # 0 75.0 210.0 259.016562
73
+ # 1 75.0 222.5 258.362212
74
+ # 2 75.0 237.5 258.318240
75
+ # 3 75.0 267.5 256.928497
76
+ # 4 75.0 285.0 261.614103
41
77
  ```
42
78
 
43
- Succinctly, we "pivot" Xarray Datasets to treat them like tables so we can run
79
+ Succinctly, we "pivot" Xarray Datasets (with consistent dimensions) to treat them like tables so we can run
44
80
  SQL queries against them.
45
81
 
46
82
  ## Why build this?
@@ -58,18 +94,18 @@ A few reasons:
58
94
 
59
95
  This is a light-weight way to prove the value of the interface.
60
96
 
61
- The larger goal is to explore the hypothesis that [Pangeo](https://pangeo.io/)
62
- is a scientific database. Here, xarray-sql can be thought of as a missing DB
63
- front end.
97
+ The larger goal is to explore the hypothesis that the [Pangeo](https://pangeo.io/)
98
+ ecosystem is a scientific database. Here, xarray-sql can be thought of as a missing
99
+ DB front end.
64
100
 
65
101
  ## How does it work?
66
102
 
67
- All chunks in an Xarray Dataset are transformed into a Dask DataFrame via
103
+ All chunks in a Xarray Dataset are transformed into a Dask DataFrame via
68
104
  `from_map()` and `to_dataframe()`. For SQL support, we just use `dask-sql`.
69
105
  That's it!
70
106
 
71
- _2025 update_: This library now implements a dask-like `from_map` interface in
72
- pure `datafusion` and `pyarrow`, but works with the same principle!
107
+ _2025 update_: This library now implements a Dask-like `from_map` interface in
108
+ pure DataFusion and PyArrow, but works with the same principle!
73
109
 
74
110
  ## Why does this work?
75
111
 
@@ -77,14 +113,15 @@ Underneath Xarray, Dask, and Pandas, there are NumPy arrays. These are paged in
77
113
  chunks and represented contiguously in memory. It is only a matter of metadata
78
114
  that breaks them up into ndarrays. `to_dataframe()`
79
115
  just changes this metadata (via a `ravel()`/`reshape()`), back into a column
80
- amenable to a DataFrame.
81
-
82
- There is added overhead from duplicating dimensions as columns, which we see as
83
- worth the convenience of DataFrames.
116
+ amenable to a DataFrame. We take advantage of this light weight metadata change to
117
+ make chunked information scannable by a DB engine (DataFusion).
84
118
 
85
119
  ## What are the current limitations?
86
120
 
87
- _2025 update_: TBD, `datafusion` provides a whole new world!
121
+ _2025 update_: TBD, DataFusion provides a whole new world! Currently, we're looking for
122
+ early users – "tire kickers", if you will. We'd love your input to shape the direction of this
123
+ project! Please, give this a try and [file issues](https://github.com/alxmrs/xarray-sql/issues) as
124
+ you see fit. Check out our [contributing guide](CONTRIBUTING.md), too 😉.
88
125
 
89
126
  ## What would a deeper integration look like?
90
127
 
@@ -99,6 +136,13 @@ parquet would open up integrations to numerous tools like dask, pyarrow, duckdb,
99
136
  and BigQuery. More thoughts on this
100
137
  in [#4](https://github.com/alxmrs/xarray-sql/issues/4).
101
138
 
139
+ _2025 update_: Something like this is being built across a few projects! The ones I know about are:
140
+ - [CartoDB's Raquet](https://github.com/CartoDB/raquet)
141
+ - The DataFusion community's [arrow-zarr](https://github.com/datafusion-contrib/arrow-zarr)
142
+
143
+ As of writing, this project is [amid integrating](https://github.com/alxmrs/xarray-sql/pull/69) a
144
+ rust-based DataFusion backend provided by arrow-zarr.
145
+
102
146
  ## Sponsors & Contributors
103
147
 
104
148
  I want to give a special thanks to the following folks and institutions:
@@ -108,6 +152,9 @@ I want to give a special thanks to the following folks and institutions:
108
152
  - Jake Wall and AI2/Ecoscope for compute resources and key use cases.
109
153
  - Charles Stern, Stephan Hoyer, Alexander Kmoch, Wei Ji, and Qiusheng Wu
110
154
  for the early review and discussion of this project.
155
+ - Tom Nichols, Kyle Barron, Tom White, and Maxime Dion for the [Array Working
156
+ Group](https://discourse.pangeo.io/t/new-working-group-for-distributed-array-computing/2734)
157
+ and DataFusion-specific collaboration.
111
158
 
112
159
  ## License
113
160
 
@@ -5,8 +5,9 @@ import numpy as np
5
5
  import pandas as pd
6
6
  import pyarrow as pa
7
7
  import xarray as xr
8
+ from datafusion.context import ArrowStreamExportable
8
9
 
9
- Block = t.Dict[str, slice]
10
+ Block = t.Dict[t.Hashable, slice]
10
11
  Chunks = t.Optional[t.Dict[str, int]]
11
12
 
12
13
 
@@ -54,7 +55,42 @@ def explode(ds: xr.Dataset, chunks: Chunks = None) -> t.Iterator[xr.Dataset]:
54
55
 
55
56
 
56
57
  def _block_len(block: Block) -> int:
57
- return np.prod([v.stop - v.start for v in block.values()])
58
+ return int(np.prod([v.stop - v.start for v in block.values()]))
59
+
60
+
61
+ def from_map_batched(
62
+ func: t.Callable[[...], pd.DataFrame],
63
+ *iterables,
64
+ args: t.Optional[t.Tuple] = None,
65
+ schema: pa.Schema = None,
66
+ **kwargs,
67
+ ) -> pa.RecordBatchReader:
68
+ """Create a PyArrow RecordBatchReader by mapping a function over iterables.
69
+
70
+ This is equivalent to dask's from_map but returns a PyArrow
71
+ RecordBatchReader that can be used with DataFusion. It iterates over
72
+ RecordBatches which are created via the `func` one-at-a-time.
73
+
74
+ Args:
75
+ func: Function to apply to each element of the iterables. Currently, the function
76
+ must return a Pandas DataFrame.
77
+ *iterables: Iterable objects to map the function over.
78
+ schema: Optional schema needed for the RecordBatchReader.
79
+ args: Additional positional arguments to pass to func.
80
+ **kwargs: Additional keyword arguments to pass to func.
81
+
82
+ Returns:
83
+ A PyArrow RecordBatchReader containing the stream of RecordBatches.
84
+ """
85
+ if args is None:
86
+ args = ()
87
+
88
+ def map_batches():
89
+ for items in zip(*iterables):
90
+ df = func(*items, *args, **kwargs)
91
+ yield pa.RecordBatch.from_pandas(df, schema=schema)
92
+
93
+ return pa.RecordBatchReader.from_batches(schema, map_batches())
58
94
 
59
95
 
60
96
  def from_map(
@@ -109,7 +145,29 @@ def from_map(
109
145
  return pa.concat_tables(results)
110
146
 
111
147
 
112
- def read_xarray(ds: xr.Dataset, chunks: Chunks = None) -> pa.Table:
148
+ def pivot(ds: xr.Dataset) -> pd.DataFrame:
149
+ """Converts an xarray Dataset to a pandas DataFrame."""
150
+ return ds.to_dataframe().reset_index()
151
+
152
+
153
+ def _parse_schema(ds) -> pa.Schema:
154
+ """Extracts a `pa.Schema` from the Dataset, treating dims and data_vars as columns."""
155
+ columns = []
156
+
157
+ for coord_name, coord_var in ds.coords.items():
158
+ # Only include dimension coordinates
159
+ if coord_name in ds.dims:
160
+ pa_type = pa.from_numpy_dtype(coord_var.dtype)
161
+ columns.append(pa.field(coord_name, pa_type))
162
+
163
+ for var_name, var in ds.data_vars.items():
164
+ pa_type = pa.from_numpy_dtype(var.dtype)
165
+ columns.append(pa.field(var_name, pa_type))
166
+
167
+ return pa.schema(columns)
168
+
169
+
170
+ def read_xarray(ds: xr.Dataset, chunks: Chunks = None) -> pa.RecordBatchReader:
113
171
  """Pivots an Xarray Dataset into a PyArrow Table, partitioned by chunks.
114
172
 
115
173
  Args:
@@ -121,14 +179,15 @@ def read_xarray(ds: xr.Dataset, chunks: Chunks = None) -> pa.Table:
121
179
  Returns:
122
180
  A PyArrow Table, which is a table representation of the input Dataset.
123
181
  """
182
+
183
+ def pivot_block(b: Block):
184
+ return pivot(ds.isel(b))
185
+
124
186
  fst = next(iter(ds.values())).dims
125
187
  assert all(
126
188
  da.dims == fst for da in ds.values()
127
189
  ), "All dimensions must be equal. Please filter data_vars in the Dataset."
128
190
 
129
- blocks = list(block_slices(ds, chunks))
130
-
131
- def pivot(b: Block) -> pd.DataFrame:
132
- return ds.isel(b).to_dataframe().reset_index()
133
-
134
- return from_map(pivot, blocks)
191
+ schema = _parse_schema(ds)
192
+ blocks = block_slices(ds, chunks)
193
+ return from_map_batched(pivot_block, blocks, schema=schema)