xml2arrow 0.17.0__tar.gz → 0.18.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {xml2arrow-0.17.0 → xml2arrow-0.18.0}/.github/workflows/CI.yml +19 -3
- {xml2arrow-0.17.0 → xml2arrow-0.18.0}/Cargo.lock +39 -44
- xml2arrow-0.18.0/Cargo.toml +25 -0
- {xml2arrow-0.17.0 → xml2arrow-0.18.0}/PKG-INFO +7 -3
- {xml2arrow-0.17.0 → xml2arrow-0.18.0}/README.md +6 -2
- {xml2arrow-0.17.0 → xml2arrow-0.18.0}/python/xml2arrow/_xml2arrow.pyi +12 -1
- {xml2arrow-0.17.0 → xml2arrow-0.18.0}/src/file_like.rs +5 -0
- {xml2arrow-0.17.0 → xml2arrow-0.18.0}/src/lib.rs +61 -9
- {xml2arrow-0.17.0 → xml2arrow-0.18.0}/tests/test_xml2arrow.py +156 -0
- xml2arrow-0.17.0/Cargo.toml +0 -18
- {xml2arrow-0.17.0 → xml2arrow-0.18.0}/.gitignore +0 -0
- {xml2arrow-0.17.0 → xml2arrow-0.18.0}/LICENSE +0 -0
- {xml2arrow-0.17.0 → xml2arrow-0.18.0}/pyproject.toml +0 -0
- {xml2arrow-0.17.0 → xml2arrow-0.18.0}/python/xml2arrow/__init__.py +0 -0
- {xml2arrow-0.17.0 → xml2arrow-0.18.0}/python/xml2arrow/exceptions.py +0 -0
- {xml2arrow-0.17.0 → xml2arrow-0.18.0}/python/xml2arrow/py.typed +0 -0
- {xml2arrow-0.17.0 → xml2arrow-0.18.0}/tests/conftest.py +0 -0
- {xml2arrow-0.17.0 → xml2arrow-0.18.0}/tests/test_data/bookstore.xml +0 -0
- {xml2arrow-0.17.0 → xml2arrow-0.18.0}/tests/test_data/bookstore.yaml +0 -0
- {xml2arrow-0.17.0 → xml2arrow-0.18.0}/tests/test_data/stations.xml +0 -0
- {xml2arrow-0.17.0 → xml2arrow-0.18.0}/tests/test_data/stations.yaml +0 -0
|
@@ -31,6 +31,20 @@ jobs:
|
|
|
31
31
|
- name: Run Ruff format
|
|
32
32
|
run: ruff format python/ --check
|
|
33
33
|
|
|
34
|
+
rust-lint:
|
|
35
|
+
name: Rust Lint
|
|
36
|
+
runs-on: ubuntu-latest
|
|
37
|
+
steps:
|
|
38
|
+
- uses: actions/checkout@v4
|
|
39
|
+
- uses: dtolnay/rust-toolchain@stable
|
|
40
|
+
with:
|
|
41
|
+
components: rustfmt,clippy
|
|
42
|
+
- uses: Swatinem/rust-cache@v2
|
|
43
|
+
- name: Run rustfmt
|
|
44
|
+
run: cargo fmt --check
|
|
45
|
+
- name: Run clippy
|
|
46
|
+
run: cargo clippy --all-targets -- -D warnings
|
|
47
|
+
|
|
34
48
|
type-check:
|
|
35
49
|
name: Type Check
|
|
36
50
|
needs: lint
|
|
@@ -47,7 +61,7 @@ jobs:
|
|
|
47
61
|
|
|
48
62
|
test:
|
|
49
63
|
name: Test
|
|
50
|
-
needs: lint # Only run tests if linting passes
|
|
64
|
+
needs: [lint, rust-lint] # Only run tests if linting passes
|
|
51
65
|
runs-on: ${{ matrix.os }}
|
|
52
66
|
strategy:
|
|
53
67
|
fail-fast: false
|
|
@@ -62,11 +76,13 @@ jobs:
|
|
|
62
76
|
- name: Install Rust toolchain
|
|
63
77
|
uses: dtolnay/rust-toolchain@stable
|
|
64
78
|
- name: Install dependencies
|
|
65
|
-
run: pip install maturin pytest pytest-cov
|
|
79
|
+
run: pip install maturin pytest pytest-cov pyarrow
|
|
66
80
|
- name: Build wheel
|
|
67
81
|
run: maturin build --release --out dist
|
|
68
82
|
- name: Install wheel
|
|
69
|
-
|
|
83
|
+
# --no-index forces the just-built wheel; without it pip may pick a
|
|
84
|
+
# released version from PyPI and silently test the wrong artifact.
|
|
85
|
+
run: pip install --no-index --find-links=dist xml2arrow
|
|
70
86
|
- name: Run tests with coverage
|
|
71
87
|
run: pytest tests/ -v --cov=xml2arrow --cov-report=xml --cov-report=term
|
|
72
88
|
- name: Upload coverage to Codecov
|
|
@@ -4,7 +4,7 @@ version = 4
|
|
|
4
4
|
|
|
5
5
|
[[package]]
|
|
6
6
|
name = "_xml2arrow"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.18.0"
|
|
8
8
|
dependencies = [
|
|
9
9
|
"arrow",
|
|
10
10
|
"pyo3",
|
|
@@ -51,9 +51,9 @@ dependencies = [
|
|
|
51
51
|
|
|
52
52
|
[[package]]
|
|
53
53
|
name = "arrow"
|
|
54
|
-
version = "
|
|
54
|
+
version = "59.0.0"
|
|
55
55
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
56
|
-
checksum = "
|
|
56
|
+
checksum = "ffaaa3e009861fd829d0a24dd6f115aa8e4634324bb092147d43baafe69ca4a7"
|
|
57
57
|
dependencies = [
|
|
58
58
|
"arrow-arith",
|
|
59
59
|
"arrow-array",
|
|
@@ -73,9 +73,9 @@ dependencies = [
|
|
|
73
73
|
|
|
74
74
|
[[package]]
|
|
75
75
|
name = "arrow-arith"
|
|
76
|
-
version = "
|
|
76
|
+
version = "59.0.0"
|
|
77
77
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
78
|
-
checksum = "
|
|
78
|
+
checksum = "3ac95125e1d71c4a252b5a9c729aef111e80418f08aaa6dbabd1ba66918247fc"
|
|
79
79
|
dependencies = [
|
|
80
80
|
"arrow-array",
|
|
81
81
|
"arrow-buffer",
|
|
@@ -87,9 +87,9 @@ dependencies = [
|
|
|
87
87
|
|
|
88
88
|
[[package]]
|
|
89
89
|
name = "arrow-array"
|
|
90
|
-
version = "
|
|
90
|
+
version = "59.0.0"
|
|
91
91
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
92
|
-
checksum = "
|
|
92
|
+
checksum = "0c60c79628e9a97cb90d7a0dc3e944f216a902f837d4ecabc14d524bddbbc137"
|
|
93
93
|
dependencies = [
|
|
94
94
|
"ahash",
|
|
95
95
|
"arrow-buffer",
|
|
@@ -97,7 +97,7 @@ dependencies = [
|
|
|
97
97
|
"arrow-schema",
|
|
98
98
|
"chrono",
|
|
99
99
|
"half",
|
|
100
|
-
"hashbrown
|
|
100
|
+
"hashbrown",
|
|
101
101
|
"num-complex",
|
|
102
102
|
"num-integer",
|
|
103
103
|
"num-traits",
|
|
@@ -105,9 +105,9 @@ dependencies = [
|
|
|
105
105
|
|
|
106
106
|
[[package]]
|
|
107
107
|
name = "arrow-buffer"
|
|
108
|
-
version = "
|
|
108
|
+
version = "59.0.0"
|
|
109
109
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
110
|
-
checksum = "
|
|
110
|
+
checksum = "6026f638c400e9878c1b1cc05c3cfd46fbf381285916ab408678701c1df46c1a"
|
|
111
111
|
dependencies = [
|
|
112
112
|
"bytes",
|
|
113
113
|
"half",
|
|
@@ -117,9 +117,9 @@ dependencies = [
|
|
|
117
117
|
|
|
118
118
|
[[package]]
|
|
119
119
|
name = "arrow-cast"
|
|
120
|
-
version = "
|
|
120
|
+
version = "59.0.0"
|
|
121
121
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
122
|
-
checksum = "
|
|
122
|
+
checksum = "c82c236c3caf8df5664284f3f1fbe89938852163998c3fdbf37e84ac220445e9"
|
|
123
123
|
dependencies = [
|
|
124
124
|
"arrow-array",
|
|
125
125
|
"arrow-buffer",
|
|
@@ -138,9 +138,9 @@ dependencies = [
|
|
|
138
138
|
|
|
139
139
|
[[package]]
|
|
140
140
|
name = "arrow-csv"
|
|
141
|
-
version = "
|
|
141
|
+
version = "59.0.0"
|
|
142
142
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
143
|
-
checksum = "
|
|
143
|
+
checksum = "12714e5fb7954159af1e26d4e0d37108bcf1a2ad5ee5c5bf02a944d564d588b7"
|
|
144
144
|
dependencies = [
|
|
145
145
|
"arrow-array",
|
|
146
146
|
"arrow-cast",
|
|
@@ -153,9 +153,9 @@ dependencies = [
|
|
|
153
153
|
|
|
154
154
|
[[package]]
|
|
155
155
|
name = "arrow-data"
|
|
156
|
-
version = "
|
|
156
|
+
version = "59.0.0"
|
|
157
157
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
158
|
-
checksum = "
|
|
158
|
+
checksum = "7bd568aa70c4ec5947027b0d5caee94877433b661a0bb9e8ddceeeb5f0c9b1ab"
|
|
159
159
|
dependencies = [
|
|
160
160
|
"arrow-buffer",
|
|
161
161
|
"arrow-schema",
|
|
@@ -166,9 +166,9 @@ dependencies = [
|
|
|
166
166
|
|
|
167
167
|
[[package]]
|
|
168
168
|
name = "arrow-ipc"
|
|
169
|
-
version = "
|
|
169
|
+
version = "59.0.0"
|
|
170
170
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
171
|
-
checksum = "
|
|
171
|
+
checksum = "e57ee4d470eab1a021bc4b63fa2b2c15d572892bf227b0a982d3b755a6c662b5"
|
|
172
172
|
dependencies = [
|
|
173
173
|
"arrow-array",
|
|
174
174
|
"arrow-buffer",
|
|
@@ -180,15 +180,16 @@ dependencies = [
|
|
|
180
180
|
|
|
181
181
|
[[package]]
|
|
182
182
|
name = "arrow-json"
|
|
183
|
-
version = "
|
|
183
|
+
version = "59.0.0"
|
|
184
184
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
185
|
-
checksum = "
|
|
185
|
+
checksum = "38f47e0e7a284e1f3707a780dc8cd5451b1614e9e398ea2d9ca03c7a2fe9a9ed"
|
|
186
186
|
dependencies = [
|
|
187
187
|
"arrow-array",
|
|
188
188
|
"arrow-buffer",
|
|
189
189
|
"arrow-cast",
|
|
190
|
-
"arrow-
|
|
190
|
+
"arrow-ord",
|
|
191
191
|
"arrow-schema",
|
|
192
|
+
"arrow-select",
|
|
192
193
|
"chrono",
|
|
193
194
|
"half",
|
|
194
195
|
"indexmap",
|
|
@@ -204,9 +205,9 @@ dependencies = [
|
|
|
204
205
|
|
|
205
206
|
[[package]]
|
|
206
207
|
name = "arrow-ord"
|
|
207
|
-
version = "
|
|
208
|
+
version = "59.0.0"
|
|
208
209
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
209
|
-
checksum = "
|
|
210
|
+
checksum = "a79cf73ad2eba8686ec2aa9bbf8671208e509025f166afc040cedbd94ffe4983"
|
|
210
211
|
dependencies = [
|
|
211
212
|
"arrow-array",
|
|
212
213
|
"arrow-buffer",
|
|
@@ -217,9 +218,9 @@ dependencies = [
|
|
|
217
218
|
|
|
218
219
|
[[package]]
|
|
219
220
|
name = "arrow-pyarrow"
|
|
220
|
-
version = "
|
|
221
|
+
version = "59.0.0"
|
|
221
222
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
222
|
-
checksum = "
|
|
223
|
+
checksum = "b9e66bd74e4a47c6df9a2af16d9cca97e37cdfe5b3207d5e3558646e07957a27"
|
|
223
224
|
dependencies = [
|
|
224
225
|
"arrow-array",
|
|
225
226
|
"arrow-data",
|
|
@@ -229,9 +230,9 @@ dependencies = [
|
|
|
229
230
|
|
|
230
231
|
[[package]]
|
|
231
232
|
name = "arrow-row"
|
|
232
|
-
version = "
|
|
233
|
+
version = "59.0.0"
|
|
233
234
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
234
|
-
checksum = "
|
|
235
|
+
checksum = "cea0f7d8ed6182f14952761e2c0f989852d5aa334fcbc49f73a9f2247c25b879"
|
|
235
236
|
dependencies = [
|
|
236
237
|
"arrow-array",
|
|
237
238
|
"arrow-buffer",
|
|
@@ -242,18 +243,18 @@ dependencies = [
|
|
|
242
243
|
|
|
243
244
|
[[package]]
|
|
244
245
|
name = "arrow-schema"
|
|
245
|
-
version = "
|
|
246
|
+
version = "59.0.0"
|
|
246
247
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
247
|
-
checksum = "
|
|
248
|
+
checksum = "80b3e786a0dd9103acd583a6fb486dbf2f3268466cc0bd571dcf34cef231c1f1"
|
|
248
249
|
dependencies = [
|
|
249
250
|
"bitflags",
|
|
250
251
|
]
|
|
251
252
|
|
|
252
253
|
[[package]]
|
|
253
254
|
name = "arrow-select"
|
|
254
|
-
version = "
|
|
255
|
+
version = "59.0.0"
|
|
255
256
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
256
|
-
checksum = "
|
|
257
|
+
checksum = "067a67e0361f6c31f4a7248759f36ca4ca71b187a941ed4d49da1c7d3d4db624"
|
|
257
258
|
dependencies = [
|
|
258
259
|
"ahash",
|
|
259
260
|
"arrow-array",
|
|
@@ -265,9 +266,9 @@ dependencies = [
|
|
|
265
266
|
|
|
266
267
|
[[package]]
|
|
267
268
|
name = "arrow-string"
|
|
268
|
-
version = "
|
|
269
|
+
version = "59.0.0"
|
|
269
270
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
270
|
-
checksum = "
|
|
271
|
+
checksum = "99bc95847f3ff62a2b03d6f8ce2e3e78f01362060549a2a311898dd442f6256d"
|
|
271
272
|
dependencies = [
|
|
272
273
|
"arrow-array",
|
|
273
274
|
"arrow-buffer",
|
|
@@ -479,12 +480,6 @@ dependencies = [
|
|
|
479
480
|
"num-traits",
|
|
480
481
|
]
|
|
481
482
|
|
|
482
|
-
[[package]]
|
|
483
|
-
name = "hashbrown"
|
|
484
|
-
version = "0.16.0"
|
|
485
|
-
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
486
|
-
checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d"
|
|
487
|
-
|
|
488
483
|
[[package]]
|
|
489
484
|
name = "hashbrown"
|
|
490
485
|
version = "0.17.0"
|
|
@@ -527,7 +522,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
|
527
522
|
checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9"
|
|
528
523
|
dependencies = [
|
|
529
524
|
"equivalent",
|
|
530
|
-
"hashbrown
|
|
525
|
+
"hashbrown",
|
|
531
526
|
]
|
|
532
527
|
|
|
533
528
|
[[package]]
|
|
@@ -752,9 +747,9 @@ dependencies = [
|
|
|
752
747
|
|
|
753
748
|
[[package]]
|
|
754
749
|
name = "quick-xml"
|
|
755
|
-
version = "0.
|
|
750
|
+
version = "0.41.0"
|
|
756
751
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
757
|
-
checksum = "
|
|
752
|
+
checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1"
|
|
758
753
|
dependencies = [
|
|
759
754
|
"memchr",
|
|
760
755
|
]
|
|
@@ -1051,9 +1046,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
|
|
|
1051
1046
|
|
|
1052
1047
|
[[package]]
|
|
1053
1048
|
name = "xml2arrow"
|
|
1054
|
-
version = "0.
|
|
1049
|
+
version = "0.18.0"
|
|
1055
1050
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1056
|
-
checksum = "
|
|
1051
|
+
checksum = "966e16865b1dc8abf5f1f4c4cb1600a5f36fc3382ddcf053ba2ae65def571ff3"
|
|
1057
1052
|
dependencies = [
|
|
1058
1053
|
"arrow",
|
|
1059
1054
|
"atoi",
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
[package]
|
|
2
|
+
name = "_xml2arrow"
|
|
3
|
+
version = "0.18.0"
|
|
4
|
+
edition = "2024"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
|
|
7
|
+
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
|
8
|
+
[lib]
|
|
9
|
+
name = "_xml2arrow"
|
|
10
|
+
crate-type = ["cdylib"]
|
|
11
|
+
|
|
12
|
+
[dependencies]
|
|
13
|
+
xml2arrow = { version = "0.18.0", features = ["python"] }
|
|
14
|
+
arrow = { version = "59.0.0", features = ["pyarrow"] }
|
|
15
|
+
|
|
16
|
+
[dependencies.pyo3]
|
|
17
|
+
version = "0.28"
|
|
18
|
+
features = ["extension-module", "abi3-py310"]
|
|
19
|
+
|
|
20
|
+
# Wheel builds go through this profile (maturin build --release); the
|
|
21
|
+
# size/speed win is worth the slower release compile.
|
|
22
|
+
[profile.release]
|
|
23
|
+
lto = true
|
|
24
|
+
codegen-units = 1
|
|
25
|
+
strip = "symbols"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: xml2arrow
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.18.0
|
|
4
4
|
Classifier: Programming Language :: Rust
|
|
5
5
|
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
6
6
|
Classifier: License :: OSI Approved :: MIT License
|
|
@@ -48,7 +48,10 @@ Rust crate for high performance.
|
|
|
48
48
|
- 🎯 **Type conversion** including automatic scale and offset transforms for float fields
|
|
49
49
|
- 💡 **Attribute and element extraction** using `@`-prefixed path segments for attributes
|
|
50
50
|
- ⏹️ **Early termination** via `stop_at_paths` for efficiently reading only part of a file
|
|
51
|
-
- 🐍 **Flexible input** — accepts file paths, path-like objects,
|
|
51
|
+
- 🐍 **Flexible input** — accepts file paths, path-like objects, readable file-like
|
|
52
|
+
objects, or in-memory `bytes`/`bytearray` (parsed zero-copy, no intermediate buffering)
|
|
53
|
+
- 🧵 **Thread-friendly** — the GIL is released while parsing, so threads sharing one
|
|
54
|
+
parser instance can parse multiple documents in parallel
|
|
52
55
|
|
|
53
56
|
## Installation
|
|
54
57
|
|
|
@@ -137,7 +140,8 @@ import polars as pl
|
|
|
137
140
|
from xml2arrow import XmlToArrowParser
|
|
138
141
|
|
|
139
142
|
parser = XmlToArrowParser("config.yaml")
|
|
140
|
-
record_batches = parser.parse("data.xml") # also accepts pathlib.Path
|
|
143
|
+
record_batches = parser.parse("data.xml") # also accepts pathlib.Path, bytes,
|
|
144
|
+
# bytearray, or any file-like object
|
|
141
145
|
|
|
142
146
|
# Access a table by name
|
|
143
147
|
batch = record_batches["measurements"] # pyarrow.RecordBatch
|
|
@@ -19,7 +19,10 @@ Rust crate for high performance.
|
|
|
19
19
|
- 🎯 **Type conversion** including automatic scale and offset transforms for float fields
|
|
20
20
|
- 💡 **Attribute and element extraction** using `@`-prefixed path segments for attributes
|
|
21
21
|
- ⏹️ **Early termination** via `stop_at_paths` for efficiently reading only part of a file
|
|
22
|
-
- 🐍 **Flexible input** — accepts file paths, path-like objects,
|
|
22
|
+
- 🐍 **Flexible input** — accepts file paths, path-like objects, readable file-like
|
|
23
|
+
objects, or in-memory `bytes`/`bytearray` (parsed zero-copy, no intermediate buffering)
|
|
24
|
+
- 🧵 **Thread-friendly** — the GIL is released while parsing, so threads sharing one
|
|
25
|
+
parser instance can parse multiple documents in parallel
|
|
23
26
|
|
|
24
27
|
## Installation
|
|
25
28
|
|
|
@@ -108,7 +111,8 @@ import polars as pl
|
|
|
108
111
|
from xml2arrow import XmlToArrowParser
|
|
109
112
|
|
|
110
113
|
parser = XmlToArrowParser("config.yaml")
|
|
111
|
-
record_batches = parser.parse("data.xml") # also accepts pathlib.Path
|
|
114
|
+
record_batches = parser.parse("data.xml") # also accepts pathlib.Path, bytes,
|
|
115
|
+
# bytearray, or any file-like object
|
|
112
116
|
|
|
113
117
|
# Access a table by name
|
|
114
118
|
batch = record_batches["measurements"] # pyarrow.RecordBatch
|
|
@@ -24,6 +24,8 @@ class XmlToArrowParser:
|
|
|
24
24
|
config_path: The path to the YAML configuration file.
|
|
25
25
|
|
|
26
26
|
Raises:
|
|
27
|
+
OSError: If the configuration file cannot be opened (e.g.
|
|
28
|
+
FileNotFoundError); the message includes the path.
|
|
27
29
|
Xml2ArrowError: If the configuration file cannot be loaded, parsed, or
|
|
28
30
|
validated. Because validation happens at construction time, an
|
|
29
31
|
invalid config (e.g. InvalidConfigError) is raised here rather
|
|
@@ -38,6 +40,8 @@ class XmlToArrowParser:
|
|
|
38
40
|
|
|
39
41
|
In-memory inputs (``bytes`` and ``bytearray``) take a zero-copy fast
|
|
40
42
|
path. Paths and file-like objects stream through a buffered reader.
|
|
43
|
+
The GIL is released while parsing, so threads sharing one parser
|
|
44
|
+
instance can parse different sources in parallel.
|
|
41
45
|
|
|
42
46
|
Args:
|
|
43
47
|
source: The XML to parse. Accepts a path (``str`` or ``os.PathLike``),
|
|
@@ -49,9 +53,16 @@ class XmlToArrowParser:
|
|
|
49
53
|
PyArrow RecordBatch objects.
|
|
50
54
|
|
|
51
55
|
Raises:
|
|
56
|
+
OSError: If ``source`` is a path that cannot be opened (e.g.
|
|
57
|
+
FileNotFoundError); the message includes the path.
|
|
58
|
+
ValueError: If ``source`` is a ``str`` holding XML content rather
|
|
59
|
+
than a file path.
|
|
60
|
+
TypeError: If ``source`` is not a supported input type.
|
|
52
61
|
Xml2ArrowError: If an error occurs during XML parsing or Arrow
|
|
53
62
|
table creation. This can include errors such as invalid XML,
|
|
54
|
-
incorrect configuration, or unsupported data types.
|
|
63
|
+
incorrect configuration, or unsupported data types. Exceptions
|
|
64
|
+
raised by a file-like object's ``read()`` method propagate
|
|
65
|
+
unchanged.
|
|
55
66
|
"""
|
|
56
67
|
|
|
57
68
|
def __repr__(self) -> str: ...
|
|
@@ -20,6 +20,11 @@ use pyo3::prelude::*;
|
|
|
20
20
|
use pyo3::sync::PyOnceLock;
|
|
21
21
|
|
|
22
22
|
/// A read-only adapter over a Python file-like object.
|
|
23
|
+
///
|
|
24
|
+
/// Exceptions raised inside `read()` are flattened into `io::Error` here (the
|
|
25
|
+
/// `Read` interface demands it), but pyo3 stores the original `PyErr` as the
|
|
26
|
+
/// inner error and upstream's `From<Error> for PyErr` recovers it, so the
|
|
27
|
+
/// caller still sees the real exception rather than a stringified wrapper.
|
|
23
28
|
pub struct PyBinaryFile {
|
|
24
29
|
inner: Py<PyAny>,
|
|
25
30
|
is_text: bool,
|
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
use arrow::pyarrow::ToPyArrow;
|
|
2
2
|
use pyo3::{
|
|
3
|
+
exceptions::{PyOSError, PyValueError},
|
|
3
4
|
prelude::*,
|
|
4
5
|
types::{PyByteArray, PyBytes, PyDict},
|
|
5
6
|
};
|
|
6
7
|
use std::fs::File;
|
|
7
8
|
use std::io::{BufReader, Read};
|
|
8
|
-
use std::path::PathBuf;
|
|
9
|
+
use std::path::{Path, PathBuf};
|
|
9
10
|
use xml2arrow::Parser;
|
|
10
11
|
use xml2arrow::config::Config;
|
|
11
12
|
use xml2arrow::errors::{
|
|
@@ -23,6 +24,41 @@ fn _get_version() -> &'static str {
|
|
|
23
24
|
VERSION
|
|
24
25
|
}
|
|
25
26
|
|
|
27
|
+
/// Rebuilds a filesystem error as a Python `OSError` that carries the path;
|
|
28
|
+
/// the raw `io::Error` drops it, which made "No such file or directory
|
|
29
|
+
/// (os error 2)" unactionable. Also catches the common mistake of passing
|
|
30
|
+
/// XML *content* as a `str`, which would otherwise surface as a baffling
|
|
31
|
+
/// `FileNotFoundError` (or `ENAMETOOLONG` for larger documents).
|
|
32
|
+
fn open_error(err: std::io::Error, path: &Path) -> PyErr {
|
|
33
|
+
let text = path.to_string_lossy();
|
|
34
|
+
if text
|
|
35
|
+
.trim_start_matches('\u{feff}')
|
|
36
|
+
.trim_start()
|
|
37
|
+
.starts_with('<')
|
|
38
|
+
{
|
|
39
|
+
return PyValueError::new_err(
|
|
40
|
+
"source looks like XML content, not a file path; pass XML content as bytes \
|
|
41
|
+
(e.g. source.encode()) or wrap it in io.StringIO",
|
|
42
|
+
);
|
|
43
|
+
}
|
|
44
|
+
let Some(code) = err.raw_os_error() else {
|
|
45
|
+
return err.into();
|
|
46
|
+
};
|
|
47
|
+
// Strip io::Error's "(os error N)" suffix; OSError re-renders the code.
|
|
48
|
+
let msg = err.to_string();
|
|
49
|
+
let msg = msg.split(" (os error ").next().unwrap_or(&msg).to_string();
|
|
50
|
+
let filename = text.into_owned();
|
|
51
|
+
// OSError's multi-arg constructor picks the right subclass
|
|
52
|
+
// (FileNotFoundError, PermissionError, ...) from the error code and
|
|
53
|
+
// includes the filename in the message. On Windows `raw_os_error` is a
|
|
54
|
+
// winerror, which OSError translates only via its fourth argument.
|
|
55
|
+
#[cfg(windows)]
|
|
56
|
+
let args = (code, msg, filename, code);
|
|
57
|
+
#[cfg(not(windows))]
|
|
58
|
+
let args = (code, msg, filename);
|
|
59
|
+
PyOSError::new_err(args)
|
|
60
|
+
}
|
|
61
|
+
|
|
26
62
|
/// Represents an XML input source.
|
|
27
63
|
///
|
|
28
64
|
/// `Bytes` (zero-copy) and `OwnedBytes` (a safe copy of a mutable
|
|
@@ -46,11 +82,12 @@ impl<'a, 'py> FromPyObject<'a, 'py> for XmlInput<'py> {
|
|
|
46
82
|
if let Ok(ba) = ob.cast::<PyByteArray>() {
|
|
47
83
|
return Ok(Self::OwnedBytes(ba.to_vec()));
|
|
48
84
|
}
|
|
85
|
+
// `PathBuf` extraction accepts both `str` and `os.PathLike`.
|
|
49
86
|
if let Ok(path) = ob.extract::<PathBuf>() {
|
|
50
|
-
return
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
87
|
+
return match File::open(&path) {
|
|
88
|
+
Ok(f) => Ok(Self::File(f)),
|
|
89
|
+
Err(e) => Err(open_error(e, &path)),
|
|
90
|
+
};
|
|
54
91
|
}
|
|
55
92
|
Ok(Self::FileLike(PyBinaryFile::from_bound(ob)?))
|
|
56
93
|
}
|
|
@@ -95,6 +132,9 @@ impl XmlToArrowParser {
|
|
|
95
132
|
/// XmlToArrowParser: A new parser instance.
|
|
96
133
|
#[new]
|
|
97
134
|
pub fn new(config_path: PathBuf) -> PyResult<Self> {
|
|
135
|
+
// A missing/unreadable config should name the offending path; the
|
|
136
|
+
// io::Error that bubbles out of `from_yaml_file` drops it.
|
|
137
|
+
std::fs::metadata(&config_path).map_err(|e| open_error(e, &config_path))?;
|
|
98
138
|
// Compile the config once here. `Parser::new` also runs config
|
|
99
139
|
// validation, so an invalid config now surfaces at construction time
|
|
100
140
|
// rather than on the first `parse()` call.
|
|
@@ -109,6 +149,8 @@ impl XmlToArrowParser {
|
|
|
109
149
|
///
|
|
110
150
|
/// In-memory inputs (``bytes`` and ``bytearray``) take a zero-copy fast
|
|
111
151
|
/// path. Paths and file-like objects stream through a buffered reader.
|
|
152
|
+
/// The GIL is released while parsing, so threads sharing one parser
|
|
153
|
+
/// instance can parse different sources in parallel.
|
|
112
154
|
///
|
|
113
155
|
/// Args:
|
|
114
156
|
/// source: The XML to parse. Accepts ``str``, ``os.PathLike``,
|
|
@@ -118,11 +160,21 @@ impl XmlToArrowParser {
|
|
|
118
160
|
/// dict: A dictionary where keys are table names (strings) and values are PyArrow RecordBatch objects.
|
|
119
161
|
#[pyo3(signature = (source))]
|
|
120
162
|
pub fn parse(&self, py: Python<'_>, source: XmlInput<'_>) -> PyResult<Py<PyAny>> {
|
|
163
|
+
// Detaching from the interpreter is sound for every variant: `Bytes`
|
|
164
|
+
// is immutable, `OwnedBytes` was copied at extraction, `File` is a
|
|
165
|
+
// plain OS handle, and `FileLike` re-attaches for each read() call.
|
|
121
166
|
let batches = match source {
|
|
122
|
-
XmlInput::Bytes(b) =>
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
167
|
+
XmlInput::Bytes(b) => {
|
|
168
|
+
let bytes = b.as_bytes();
|
|
169
|
+
py.detach(|| self.parser.parse_slice(bytes))?
|
|
170
|
+
}
|
|
171
|
+
XmlInput::OwnedBytes(v) => py.detach(|| self.parser.parse_slice(&v))?,
|
|
172
|
+
XmlInput::File(f) => {
|
|
173
|
+
py.detach(|| self.parser.parse(BufReader::new(XmlReader::File(f))))?
|
|
174
|
+
}
|
|
175
|
+
XmlInput::FileLike(f) => {
|
|
176
|
+
py.detach(|| self.parser.parse(BufReader::new(XmlReader::FileLike(f))))?
|
|
177
|
+
}
|
|
126
178
|
};
|
|
127
179
|
let tables = PyDict::new(py);
|
|
128
180
|
for (name, batch) in batches {
|
|
@@ -1115,6 +1115,162 @@ tables:
|
|
|
1115
1115
|
assert batch.to_pydict()["value"] == ["Ünïcödé 中文 🌍"]
|
|
1116
1116
|
|
|
1117
1117
|
|
|
1118
|
+
def test_parse_str_path(stations_parser: XmlToArrowParser, test_data_dir: Path) -> None:
|
|
1119
|
+
"""Test that parse() accepts a plain str path, not just os.PathLike."""
|
|
1120
|
+
record_batches = stations_parser.parse(str(test_data_dir / "stations.xml"))
|
|
1121
|
+
assert "stations" in record_batches
|
|
1122
|
+
|
|
1123
|
+
|
|
1124
|
+
def test_parse_missing_file_error_includes_path(
|
|
1125
|
+
stations_parser: XmlToArrowParser, tmp_path: Path
|
|
1126
|
+
) -> None:
|
|
1127
|
+
"""Test that a nonexistent XML path raises FileNotFoundError naming the file."""
|
|
1128
|
+
missing = tmp_path / "nope.xml"
|
|
1129
|
+
with pytest.raises(FileNotFoundError, match="nope.xml"):
|
|
1130
|
+
stations_parser.parse(missing)
|
|
1131
|
+
with pytest.raises(FileNotFoundError, match="nope.xml"):
|
|
1132
|
+
stations_parser.parse(str(missing))
|
|
1133
|
+
|
|
1134
|
+
|
|
1135
|
+
def test_missing_config_file_error_includes_path(tmp_path: Path) -> None:
|
|
1136
|
+
"""Test that a nonexistent config path raises FileNotFoundError naming the file."""
|
|
1137
|
+
with pytest.raises(FileNotFoundError, match="no_config.yaml"):
|
|
1138
|
+
XmlToArrowParser(tmp_path / "no_config.yaml")
|
|
1139
|
+
|
|
1140
|
+
|
|
1141
|
+
def test_parse_xml_content_as_str_raises_helpful_error(
|
|
1142
|
+
stations_parser: XmlToArrowParser,
|
|
1143
|
+
) -> None:
|
|
1144
|
+
"""Test that passing XML content as a str gets a hint instead of FileNotFoundError."""
|
|
1145
|
+
with pytest.raises(ValueError, match=r"looks like XML content"):
|
|
1146
|
+
stations_parser.parse("<report></report>")
|
|
1147
|
+
# Leading whitespace and an XML declaration should still trigger the hint
|
|
1148
|
+
with pytest.raises(ValueError, match=r"looks like XML content"):
|
|
1149
|
+
stations_parser.parse('\n <?xml version="1.0"?><report/>')
|
|
1150
|
+
|
|
1151
|
+
|
|
1152
|
+
def test_parse_rejects_non_source_types(stations_parser: XmlToArrowParser) -> None:
|
|
1153
|
+
"""Test that unsupported source types raise TypeError with the documented message."""
|
|
1154
|
+
with pytest.raises(TypeError, match=r"path, bytes-like, or file-like"):
|
|
1155
|
+
stations_parser.parse(12345) # type: ignore[arg-type]
|
|
1156
|
+
|
|
1157
|
+
|
|
1158
|
+
def test_file_like_read_exception_propagates(stations_parser: XmlToArrowParser) -> None:
|
|
1159
|
+
"""Test that an exception raised inside a file-like's read() is re-raised as-is.
|
|
1160
|
+
|
|
1161
|
+
Before the read-error slot in PyBinaryFile, the original exception was
|
|
1162
|
+
flattened into an XmlParsingError message string.
|
|
1163
|
+
"""
|
|
1164
|
+
|
|
1165
|
+
class ExplodingReader:
|
|
1166
|
+
def read(self, _size: int) -> bytes:
|
|
1167
|
+
raise ConnectionResetError("connection lost mid-stream")
|
|
1168
|
+
|
|
1169
|
+
with pytest.raises(ConnectionResetError, match="connection lost mid-stream"):
|
|
1170
|
+
stations_parser.parse(ExplodingReader())
|
|
1171
|
+
|
|
1172
|
+
|
|
1173
|
+
def test_file_like_read_wrong_type_raises_type_error(
|
|
1174
|
+
stations_parser: XmlToArrowParser,
|
|
1175
|
+
) -> None:
|
|
1176
|
+
"""Test that a read() returning a non-buffer type surfaces as TypeError."""
|
|
1177
|
+
|
|
1178
|
+
class BadReader:
|
|
1179
|
+
def read(self, _size: int) -> int:
|
|
1180
|
+
return 42
|
|
1181
|
+
|
|
1182
|
+
with pytest.raises(TypeError):
|
|
1183
|
+
stations_parser.parse(BadReader())
|
|
1184
|
+
|
|
1185
|
+
|
|
1186
|
+
def test_parse_closed_file_raises_value_error(
|
|
1187
|
+
stations_parser: XmlToArrowParser, test_data_dir: Path
|
|
1188
|
+
) -> None:
|
|
1189
|
+
"""Test that parsing an already-closed file raises the original ValueError."""
|
|
1190
|
+
f = open(test_data_dir / "stations.xml", "rb")
|
|
1191
|
+
f.close()
|
|
1192
|
+
with pytest.raises(ValueError, match="closed file"):
|
|
1193
|
+
stations_parser.parse(f)
|
|
1194
|
+
|
|
1195
|
+
|
|
1196
|
+
def test_parse_releases_the_gil(tmp_path: Path) -> None:
|
|
1197
|
+
"""Test that parse() releases the GIL so other Python threads keep running.
|
|
1198
|
+
|
|
1199
|
+
A worker thread parses a document sized so that one parse takes ~0.3s on
|
|
1200
|
+
this machine; the main thread must be able to execute Python (several
|
|
1201
|
+
loop iterations) before the worker finishes. If parse() held the GIL
|
|
1202
|
+
throughout, the main thread would freeze until the parse completed,
|
|
1203
|
+
accumulating at most the 1-2 iterations it can squeeze in before the
|
|
1204
|
+
worker enters the extension call.
|
|
1205
|
+
|
|
1206
|
+
The document size is calibrated rather than hardcoded: a release wheel
|
|
1207
|
+
parses roughly an order of magnitude faster than a debug build, and CI
|
|
1208
|
+
runners add sleep-granularity noise, so a fixed size either starves the
|
|
1209
|
+
counter on fast machines or wastes time on slow ones.
|
|
1210
|
+
"""
|
|
1211
|
+
import threading
|
|
1212
|
+
import time
|
|
1213
|
+
|
|
1214
|
+
config_path = tmp_path / "config.yaml"
|
|
1215
|
+
config_path.write_text(
|
|
1216
|
+
"""
|
|
1217
|
+
tables:
|
|
1218
|
+
- name: items
|
|
1219
|
+
xml_path: /root
|
|
1220
|
+
levels: [item]
|
|
1221
|
+
fields:
|
|
1222
|
+
- name: value
|
|
1223
|
+
xml_path: /root/item/value
|
|
1224
|
+
data_type: Int64
|
|
1225
|
+
nullable: false
|
|
1226
|
+
"""
|
|
1227
|
+
)
|
|
1228
|
+
parser = XmlToArrowParser(config_path)
|
|
1229
|
+
|
|
1230
|
+
def build_doc(items: int) -> bytes:
|
|
1231
|
+
return b"<root>" + b"<item><value>12345</value></item>" * items + b"</root>"
|
|
1232
|
+
|
|
1233
|
+
# Calibrate: measure a 200k-item parse, then scale the document so one
|
|
1234
|
+
# parse takes ~0.3s (capped at 2M items / ~70 MB to bound memory and
|
|
1235
|
+
# runtime on very slow machines).
|
|
1236
|
+
target_seconds = 0.3
|
|
1237
|
+
items = 200_000
|
|
1238
|
+
start = time.perf_counter()
|
|
1239
|
+
parser.parse(build_doc(items))
|
|
1240
|
+
duration = time.perf_counter() - start
|
|
1241
|
+
if duration < target_seconds:
|
|
1242
|
+
items = min(int(items * target_seconds / max(duration, 1e-6)), 2_000_000)
|
|
1243
|
+
big = build_doc(items)
|
|
1244
|
+
|
|
1245
|
+
started = threading.Event()
|
|
1246
|
+
done = threading.Event()
|
|
1247
|
+
worker_error: list[BaseException] = []
|
|
1248
|
+
|
|
1249
|
+
def work() -> None:
|
|
1250
|
+
started.set()
|
|
1251
|
+
try:
|
|
1252
|
+
parser.parse(big)
|
|
1253
|
+
except BaseException as exc: # pragma: no cover - only on regression
|
|
1254
|
+
worker_error.append(exc)
|
|
1255
|
+
finally:
|
|
1256
|
+
# Always set, even on failure: the main loop below must not spin
|
|
1257
|
+
# forever if the parse raises.
|
|
1258
|
+
done.set()
|
|
1259
|
+
|
|
1260
|
+
worker = threading.Thread(target=work)
|
|
1261
|
+
worker.start()
|
|
1262
|
+
assert started.wait(timeout=10)
|
|
1263
|
+
iterations = 0
|
|
1264
|
+
deadline = time.monotonic() + 60
|
|
1265
|
+
while not done.is_set() and time.monotonic() < deadline:
|
|
1266
|
+
iterations += 1
|
|
1267
|
+
time.sleep(0.001)
|
|
1268
|
+
worker.join(timeout=10)
|
|
1269
|
+
assert done.is_set(), "worker never finished parsing"
|
|
1270
|
+
assert not worker_error, f"parse() raised in the worker: {worker_error[0]!r}"
|
|
1271
|
+
assert iterations >= 3, "main thread was starved: parse() appears to hold the GIL"
|
|
1272
|
+
|
|
1273
|
+
|
|
1118
1274
|
def test_version_returns_string() -> None:
|
|
1119
1275
|
"""Test that the package version is a non-empty string.
|
|
1120
1276
|
|
xml2arrow-0.17.0/Cargo.toml
DELETED
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
[package]
|
|
2
|
-
name = "_xml2arrow"
|
|
3
|
-
version = "0.17.0"
|
|
4
|
-
edition = "2024"
|
|
5
|
-
readme = "README.md"
|
|
6
|
-
|
|
7
|
-
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
|
8
|
-
[lib]
|
|
9
|
-
name = "_xml2arrow"
|
|
10
|
-
crate-type = ["cdylib"]
|
|
11
|
-
|
|
12
|
-
[dependencies]
|
|
13
|
-
xml2arrow = { version = "0.17.0", features = ["python"] }
|
|
14
|
-
arrow = { version = "58.1.0", features = ["pyarrow"] }
|
|
15
|
-
|
|
16
|
-
[dependencies.pyo3]
|
|
17
|
-
version = "0.28"
|
|
18
|
-
features = ["extension-module", "abi3-py310"]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|