readstat-arrow 2026.9.21.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. readstat_arrow-2026.9.21.1/LICENSE +26 -0
  2. readstat_arrow-2026.9.21.1/MANIFEST.in +18 -0
  3. readstat_arrow-2026.9.21.1/PKG-INFO +247 -0
  4. readstat_arrow-2026.9.21.1/README.md +227 -0
  5. readstat_arrow-2026.9.21.1/pyproject.toml +143 -0
  6. readstat_arrow-2026.9.21.1/setup.cfg +4 -0
  7. readstat_arrow-2026.9.21.1/setup.py +93 -0
  8. readstat_arrow-2026.9.21.1/src/readstat_arrow/__init__.py +48 -0
  9. readstat_arrow-2026.9.21.1/src/readstat_arrow/_cython/__init__.py +6 -0
  10. readstat_arrow-2026.9.21.1/src/readstat_arrow/_cython/parser.py +1177 -0
  11. readstat_arrow-2026.9.21.1/src/readstat_arrow/_cython/readstat.pxd +245 -0
  12. readstat_arrow-2026.9.21.1/src/readstat_arrow/_cython/writer.py +517 -0
  13. readstat_arrow-2026.9.21.1/src/readstat_arrow/_dates.py +231 -0
  14. readstat_arrow-2026.9.21.1/src/readstat_arrow/_formats.py +5 -0
  15. readstat_arrow-2026.9.21.1/src/readstat_arrow/errors.py +14 -0
  16. readstat_arrow-2026.9.21.1/src/readstat_arrow/metadata.py +151 -0
  17. readstat_arrow-2026.9.21.1/src/readstat_arrow/py.typed +0 -0
  18. readstat_arrow-2026.9.21.1/src/readstat_arrow/reader.py +397 -0
  19. readstat_arrow-2026.9.21.1/src/readstat_arrow/writer.py +818 -0
  20. readstat_arrow-2026.9.21.1/src/readstat_arrow.egg-info/PKG-INFO +247 -0
  21. readstat_arrow-2026.9.21.1/src/readstat_arrow.egg-info/SOURCES.txt +120 -0
  22. readstat_arrow-2026.9.21.1/src/readstat_arrow.egg-info/dependency_links.txt +1 -0
  23. readstat_arrow-2026.9.21.1/src/readstat_arrow.egg-info/requires.txt +1 -0
  24. readstat_arrow-2026.9.21.1/src/readstat_arrow.egg-info/top_level.txt +1 -0
  25. readstat_arrow-2026.9.21.1/tests/data/README.md +31 -0
  26. readstat_arrow-2026.9.21.1/tests/data/hebrews.sav +0 -0
  27. readstat_arrow-2026.9.21.1/tests/data/missing_char.sav +0 -0
  28. readstat_arrow-2026.9.21.1/tests/data/missing_test.dta +0 -0
  29. readstat_arrow-2026.9.21.1/tests/data/sample.csv +6 -0
  30. readstat_arrow-2026.9.21.1/tests/data/sample.dta +0 -0
  31. readstat_arrow-2026.9.21.1/tests/data/sample.sav +0 -0
  32. readstat_arrow-2026.9.21.1/tests/data/sample.zsav +0 -0
  33. readstat_arrow-2026.9.21.1/tests/data/sample_missing.sav +0 -0
  34. readstat_arrow-2026.9.21.1/tests/data/simple_alltypes.sav +0 -0
  35. readstat_arrow-2026.9.21.1/tests/data/tegulu.sav +0 -0
  36. readstat_arrow-2026.9.21.1/tests/data/test_width.sav +0 -0
  37. readstat_arrow-2026.9.21.1/tests/test_metadata.py +133 -0
  38. readstat_arrow-2026.9.21.1/tests/test_narrowing.py +201 -0
  39. readstat_arrow-2026.9.21.1/tests/test_read.py +474 -0
  40. readstat_arrow-2026.9.21.1/tests/test_write.py +1027 -0
  41. readstat_arrow-2026.9.21.1/vendor/README.md +56 -0
  42. readstat_arrow-2026.9.21.1/vendor/ReadStat/LICENSE +19 -0
  43. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/CKHashTable.c +309 -0
  44. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/CKHashTable.h +37 -0
  45. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/readstat.h +630 -0
  46. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/readstat_bits.c +69 -0
  47. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/readstat_bits.h +20 -0
  48. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/readstat_convert.c +36 -0
  49. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/readstat_convert.h +2 -0
  50. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/readstat_error.c +129 -0
  51. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/readstat_iconv.h +15 -0
  52. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/readstat_io_unistd.c +147 -0
  53. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/readstat_io_unistd.h +11 -0
  54. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/readstat_malloc.c +34 -0
  55. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/readstat_malloc.h +4 -0
  56. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/readstat_metadata.c +53 -0
  57. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/readstat_parser.c +121 -0
  58. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/readstat_strings.h +6 -0
  59. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/readstat_value.c +178 -0
  60. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/readstat_variable.c +130 -0
  61. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/readstat_writer.c +687 -0
  62. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/readstat_writer.h +21 -0
  63. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/sas/ieee.c +420 -0
  64. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/sas/ieee.h +6 -0
  65. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/sas/readstat_sas.c +533 -0
  66. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/sas/readstat_sas.h +154 -0
  67. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/sas/readstat_sas7bcat_read.c +525 -0
  68. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/sas/readstat_sas7bcat_write.c +224 -0
  69. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/sas/readstat_sas7bdat_read.c +1575 -0
  70. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/sas/readstat_sas7bdat_write.c +812 -0
  71. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/sas/readstat_sas_rle.c +286 -0
  72. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/sas/readstat_sas_rle.h +8 -0
  73. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/sas/readstat_xport.c +28 -0
  74. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/sas/readstat_xport.h +47 -0
  75. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/sas/readstat_xport_parse_format.c +265 -0
  76. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/sas/readstat_xport_parse_format.h +4 -0
  77. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/sas/readstat_xport_parse_format.rl +68 -0
  78. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/sas/readstat_xport_read.c +789 -0
  79. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/sas/readstat_xport_write.c +596 -0
  80. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/spss/readstat_por.c +172 -0
  81. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/spss/readstat_por.h +41 -0
  82. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/spss/readstat_por_parse.c +333 -0
  83. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/spss/readstat_por_parse.h +6 -0
  84. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/spss/readstat_por_parse.rl +98 -0
  85. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/spss/readstat_por_read.c +916 -0
  86. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/spss/readstat_por_write.c +794 -0
  87. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/spss/readstat_sav.c +115 -0
  88. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/spss/readstat_sav.h +146 -0
  89. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/spss/readstat_sav_compress.c +139 -0
  90. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/spss/readstat_sav_compress.h +27 -0
  91. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/spss/readstat_sav_parse.c +843 -0
  92. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/spss/readstat_sav_parse.h +7 -0
  93. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/spss/readstat_sav_parse.rl +256 -0
  94. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/spss/readstat_sav_parse_mr_name.c +524 -0
  95. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/spss/readstat_sav_parse_mr_name.h +9 -0
  96. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/spss/readstat_sav_parse_mr_name.rl +239 -0
  97. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/spss/readstat_sav_parse_timestamp.c +684 -0
  98. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/spss/readstat_sav_parse_timestamp.h +5 -0
  99. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/spss/readstat_sav_parse_timestamp.rl +126 -0
  100. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/spss/readstat_sav_read.c +1770 -0
  101. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/spss/readstat_sav_write.c +1458 -0
  102. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/spss/readstat_spss.c +270 -0
  103. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/spss/readstat_spss.h +106 -0
  104. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/spss/readstat_spss_parse.c +804 -0
  105. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/spss/readstat_spss_parse.h +2 -0
  106. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/spss/readstat_spss_parse.rl +99 -0
  107. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/spss/readstat_zsav_compress.c +108 -0
  108. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/spss/readstat_zsav_compress.h +29 -0
  109. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/spss/readstat_zsav_read.c +190 -0
  110. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/spss/readstat_zsav_read.h +3 -0
  111. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/spss/readstat_zsav_write.c +140 -0
  112. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/spss/readstat_zsav_write.h +3 -0
  113. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/stata/readstat_dta.c +317 -0
  114. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/stata/readstat_dta.h +191 -0
  115. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/stata/readstat_dta_parse_timestamp.c +439 -0
  116. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/stata/readstat_dta_parse_timestamp.h +3 -0
  117. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/stata/readstat_dta_parse_timestamp.rl +66 -0
  118. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/stata/readstat_dta_read.c +1291 -0
  119. readstat_arrow-2026.9.21.1/vendor/ReadStat/src/stata/readstat_dta_write.c +1465 -0
  120. readstat_arrow-2026.9.21.1/vendor/win-iconv/iconv.h +52 -0
  121. readstat_arrow-2026.9.21.1/vendor/win-iconv/readme.txt +20 -0
  122. readstat_arrow-2026.9.21.1/vendor/win-iconv/win_iconv.c +2155 -0
@@ -0,0 +1,26 @@
1
+ Licenses of vendored dependencies can be found in the respective sub-folders
2
+
3
+ readstat-arrow license:
4
+ ---------------------------------------------------------------------------
5
+
6
+ The MIT License (MIT)
7
+
8
+ Copyright (c) 2026 Eirik Stavestrand
9
+
10
+ Permission is hereby granted, free of charge, to any person obtaining a copy
11
+ of this software and associated documentation files (the "Software"), to deal
12
+ in the Software without restriction, including without limitation the rights
13
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
14
+ copies of the Software, and to permit persons to whom the Software is
15
+ furnished to do so, subject to the following conditions:
16
+
17
+ The above copyright notice and this permission notice shall be included in all
18
+ copies or substantial portions of the Software.
19
+
20
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
23
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
25
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
26
+ SOFTWARE.
@@ -0,0 +1,18 @@
1
+ # The sdist must be buildable on its own. setuptools picks up the vendored
2
+ # ReadStat *.c files automatically (they are listed as Extension.sources in
3
+ # setup.py) but nothing else from vendor/, so the headers they #include would
4
+ # be missing and the build would fail. Ship the vendored tree verbatim,
5
+ # exactly as it sits in git -- see vendor/README.md for the re-sync recipe.
6
+ graft vendor
7
+
8
+ # Cython declarations and the test corpus; both are needed to build and to
9
+ # verify a build from the sdist.
10
+ include src/readstat_arrow/_cython/*.pxd
11
+ graft tests
12
+
13
+ # Generated by cythonize during the build; setup.py rewrites Extension.sources
14
+ # to point at them, which is what drags them in. Never ship generated C -- the
15
+ # build regenerates it from the .py sources (cython is a build requirement).
16
+ exclude src/readstat_arrow/_cython/*.c
17
+
18
+ global-exclude __pycache__ *.py[cod] *.so *.pyd .DS_Store
@@ -0,0 +1,247 @@
1
+ Metadata-Version: 2.4
2
+ Name: readstat-arrow
3
+ Version: 2026.9.21.1
4
+ Summary: Read and write SPSS (.sav) and Stata (.dta) files as Apache Arrow tables via ReadStat
5
+ Author: Eirik Stavestrand
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/eirki/readstat-arrow
8
+ Keywords: arrow,pyarrow,spss,stata,readstat,sav,dta
9
+ Classifier: Intended Audience :: Science/Research
10
+ Classifier: Programming Language :: Python :: 3
11
+ Classifier: Programming Language :: Cython
12
+ Classifier: Topic :: Scientific/Engineering
13
+ Requires-Python: >=3.11
14
+ Description-Content-Type: text/markdown
15
+ License-File: LICENSE
16
+ License-File: vendor/ReadStat/LICENSE
17
+ License-File: vendor/win-iconv/readme.txt
18
+ Requires-Dist: pyarrow>=17
19
+ Dynamic: license-file
20
+
21
+ # readstat-arrow
22
+
23
+ Read and write SPSS (`.sav`) and Stata (`.dta`) files as [Apache Arrow](https://arrow.apache.org/)
24
+ tables.
25
+
26
+ A thin, typed Python wrapper around the excellent [ReadStat](https://github.com/WizardMac/ReadStat)
27
+ C library. Where [pyreadstat](https://github.com/Roche/pyreadstat) returns pandas data frames,
28
+ `readstat-arrow` returns a `pyarrow.Table`, which converts for free to pandas, polars, DuckDB,
29
+ Parquet and anything else that speaks Arrow.
30
+
31
+ ```python
32
+ import readstat_arrow
33
+
34
+ table, meta = readstat_arrow.read_sav("survey.sav") # -> (pyarrow.Table, Metadata)
35
+ readstat_arrow.write_dta("survey.dta", table, meta) # the same pair back out, as Stata
36
+ ```
37
+
38
+ ## Examples
39
+
40
+ ### Write an SPSS file
41
+
42
+ ```python
43
+ import pyarrow as pa
44
+ import readstat_arrow
45
+ from readstat_arrow import Metadata
46
+
47
+ table = pa.table({"id": pa.array([1, 2, 3], pa.int32()), "sex": pa.array([1, 2, None])})
48
+
49
+ meta = Metadata(
50
+ file_label="Tiny survey",
51
+ variable_labels={"id": "Respondent id", "sex": "Sex"},
52
+ value_labels={
53
+ "sex": [
54
+ {"value": 1, "label": "Male"},
55
+ {"value": 2, "label": "Female"},
56
+ {"value": 9, "label": "Unknown"},
57
+ ]
58
+ },
59
+ formats={"sex": "F1.0"},
60
+ measures={"sex": "nominal"},
61
+ storage_widths={"sex": 1},
62
+ missing_values={"sex": {"values": [9]}}, # or a range: {"lo": 90, "hi": 99}
63
+ )
64
+
65
+ readstat_arrow.write_sav("tiny.sav", table, meta)
66
+
67
+ readstat_arrow.write_sav("bare.sav", table) # no metadata at all: just the columns
68
+ ```
69
+
70
+ `Metadata` is a set of mappings from variable name to one attribute — `variable_labels`,
71
+ `value_labels`, `formats`, `storage_widths`, `display_widths`, `measures`, `alignments`,
72
+ `missing_values` — plus the file-level `file_label`, `notes` and `multiple_response_sets`. Which
73
+ columns exist, and in what order, is the Arrow schema's business. Nothing is required: a name absent
74
+ from a mapping declares nothing and the writer falls back to the format's own default, and the
75
+ metadata may be left out altogether. SPSS stores every number as a double, so `id` reads back as
76
+ `float64`, not `int32`.
77
+
78
+ ### Write a Stata file, whole or in batches
79
+
80
+ ```python
81
+ import readstat_arrow
82
+
83
+ table, meta = readstat_arrow.read_sav("survey.sav")
84
+ readstat_arrow.write_dta("survey.dta", table, meta) # a whole table in one call
85
+
86
+ # or stream it, without holding every row in memory
87
+ with readstat_arrow.DtaWriter("survey.dta", table.schema, table.num_rows, meta) as writer:
88
+ for batch in table.to_batches(max_chunksize=100_000):
89
+ writer.write_batch(batch)
90
+ ```
91
+
92
+ `DtaWriter` (and `SavWriter`) needs the final row count up front.
93
+
94
+ ### Read an SPSS file
95
+
96
+ ```python
97
+ import readstat_arrow
98
+
99
+ table, meta = readstat_arrow.read_sav("survey.sav")
100
+
101
+ table.num_rows # -> 12_345
102
+ table.schema # dates and times are already temporal types
103
+ table.column("q1")
104
+
105
+ table.column_names # -> every variable, in file order
106
+ meta.file_label # -> "2026 satisfaction survey"
107
+ meta.variable_labels["q1"] # -> "How satisfied are you ...?"
108
+ meta.value_labels["q1"] # -> [{"value": 1, "label": "Very unsatisfied"}, ...]
109
+ meta.missing_values["q1"] # -> {"values": [9]}, or {"lo": 90, "hi": 99}
110
+
111
+ table.to_pandas() # or
112
+ import polars as pl
113
+
114
+ pl.from_arrow(table)
115
+ ```
116
+
117
+ ### Read part of a Stata file
118
+
119
+ ```python
120
+ import readstat_arrow
121
+
122
+ # header only, no rows decoded: the schema a full read would give comes back
123
+ # beside the metadata
124
+ row_count, schema, meta = readstat_arrow.read_dta_metadata("panel.dta")
125
+ schema.names # -> ["id", "income", ...], the variables in file order
126
+ schema.field("income").type # -> the type read_dta would give that column
127
+
128
+ table, meta = readstat_arrow.read_dta(
129
+ "panel.dta",
130
+ columns=["id", "income"], # only these, in this order
131
+ row_offset=1_000, # skip the first 1_000 rows
132
+ row_limit=1_000, # then read at most 1_000
133
+ preserve_user_missing=True, # keep Stata's tagged missings .a-.z
134
+ )
135
+
136
+ # with preserve_user_missing every numeric column is struct<value, tag>
137
+ table.column("income")[0].as_py() # -> {"value": None, "tag": "a"}, Stata's .a
138
+ table.column("income")[1].as_py() # -> {"value": 42.0, "tag": None}, a real number
139
+ table.column("income")[2].as_py() # -> None, a plain .
140
+ ```
141
+
142
+ ### Read a big file in less memory
143
+
144
+ Where memory is the constraint, holding the whole table in it is the expensive part of reading a
145
+ file — and the type a column is stored as is often wider than its values need. A `.sav` is the worst
146
+ of it: every numeric column is a 64-bit double whatever it holds, so a survey of one-digit codes
147
+ costs 8 bytes a cell. A `.dta` has narrow types of its own — `byte`, `int`, `long`, `float` — but a
148
+ variable is only as narrow as whoever wrote the file declared it.
149
+
150
+ `scan_and_narrow_types=True` reads each column at the width its values actually need instead:
151
+
152
+ ```python
153
+ import readstat_arrow
154
+
155
+ table, meta = readstat_arrow.read_sav("big.sav", scan_and_narrow_types=True)
156
+
157
+ table.schema.field("q1").type # -> DataType(int8), where the file says double
158
+ ```
159
+
160
+ The file is parsed twice — once to measure the values, keeping none of them, then once to read them
161
+ at the widths that fit — so the trade is time for memory: about twice the wall clock of a plain read,
162
+ and no column is ever built at its stored width.
163
+
164
+ Only a type that holds the column exactly is ever chosen: integer types when every value was a whole
165
+ number, `float32` when every value round-trips through it, else `float64`. The ladder is `int8`,
166
+ `int16`, `int32`, `float32`, `float64` — there is no `int64`, which would save nothing over the
167
+ double it replaces, and no unsigned type. Strings are untouched, and a column of nothing but nulls
168
+ comes back as `int8`. Dates and times come back as the same temporal types either way: the width
169
+ they are read at only decides how much there is to convert.
170
+
171
+ ### Read from something other than a path
172
+
173
+ Every `read_*` function also takes a binary file object, so a file that arrives over the network or
174
+ out of an archive never has to be written to disk first.
175
+
176
+ ```python
177
+ import io, zipfile
178
+ import readstat_arrow
179
+
180
+ # straight out of a zip archive, without extracting it
181
+ with zipfile.ZipFile("survey.zip") as archive, archive.open("survey.sav") as member:
182
+ table, meta = readstat_arrow.read_sav(member)
183
+
184
+ # or from bytes you already have in hand
185
+ table, meta = readstat_arrow.read_sav(io.BytesIO(downloaded))
186
+
187
+ # an open file works too, and is left open where reading stopped
188
+ with open("survey.sav", "rb") as file:
189
+ row_count, schema, meta = readstat_arrow.read_sav_metadata(file)
190
+ ```
191
+
192
+ The file object must be seekable, and is read from wherever it currently is - so a `.sav` embedded
193
+ in a larger stream can be read by seeking to its first byte. The writers have taken a file object
194
+ all along.
195
+
196
+
197
+ ## Development
198
+
199
+ Requires [uv](https://docs.astral.sh/uv/) and a C compiler.
200
+
201
+ ```sh
202
+ git clone <repo-url>
203
+ cd readstat-arrow
204
+ uv sync # builds the Cython extension into .venv
205
+ uv run coverage run -m pytest && uv run coverage report || uv run coverage html
206
+ uv run ruff check . && uv run ruff format --check . && uv run mypy
207
+ ```
208
+
209
+ `uv run mypy` checks `src/` and `tests/` in strict mode. The Cython sources in
210
+ `src/readstat_arrow/_cython/` are excluded — they are typed for Cython's C type system, which mypy
211
+ cannot follow, and Cython checks them at compile time.
212
+
213
+ ReadStat is vendored as a git submodule at `vendor/ReadStat`; bump it with `git submodule update
214
+ --remote vendor/ReadStat`. `uv sync` rebuilds the extension whenever `_cython/`, `setup.py` or the
215
+ ReadStat sources change (see `[tool.uv] cache-keys` in `pyproject.toml`).
216
+
217
+ ## Layout
218
+
219
+ ```
220
+ pyproject.toml project metadata, deps, tool config (uv/ruff/mypy/pytest)
221
+ setup.py Cython extension definition (compiles ReadStat in)
222
+ vendor/ReadStat/ git submodule
223
+ src/readstat_arrow/
224
+ __init__.py public API re-exports
225
+ reader.py read_* and read_*_metadata functions, table assembly, type narrowing
226
+ writer.py SavWriter / DtaWriter, write_* functions, type planning
227
+ metadata.py the Metadata dataclass and its per-variable mappings
228
+ errors.py ReadstatError, ReadstatWarning
229
+ _formats.py FileFormat literal type
230
+ _dates.py display-format -> temporal type conversion
231
+ _cython/ everything Cython compiles (private)
232
+ parser.py pure-Python-mode Cython: ReadStat callbacks -> Arrow buffers
233
+ writer.py pure-Python-mode Cython: Arrow buffers -> readstat_insert_*
234
+ readstat.pxd C declarations for readstat.h
235
+ tests/ pytest suite; sample files under tests/data/
236
+ ```
237
+
238
+ ## Versioning
239
+
240
+ Releases use [CalVer](https://calver.org/) in the form `YYYY.MM.DD.INC0` (e.g. `2026.09.01.0`, then
241
+ `2026.09.01.1` for a fix in the same dat). There are no compatibility promises encoded in the
242
+ number. The version is set once in `pyproject.toml` and exposed as `readstat_arrow.__version__`.
243
+
244
+ ## Licence
245
+
246
+ MIT. ReadStat is MIT-licensed; the sample files under `tests/data/` come from pyreadstat (Apache
247
+ 2.0) — see `tests/data/README.md`.
@@ -0,0 +1,227 @@
1
+ # readstat-arrow
2
+
3
+ Read and write SPSS (`.sav`) and Stata (`.dta`) files as [Apache Arrow](https://arrow.apache.org/)
4
+ tables.
5
+
6
+ A thin, typed Python wrapper around the excellent [ReadStat](https://github.com/WizardMac/ReadStat)
7
+ C library. Where [pyreadstat](https://github.com/Roche/pyreadstat) returns pandas data frames,
8
+ `readstat-arrow` returns a `pyarrow.Table`, which converts for free to pandas, polars, DuckDB,
9
+ Parquet and anything else that speaks Arrow.
10
+
11
+ ```python
12
+ import readstat_arrow
13
+
14
+ table, meta = readstat_arrow.read_sav("survey.sav") # -> (pyarrow.Table, Metadata)
15
+ readstat_arrow.write_dta("survey.dta", table, meta) # the same pair back out, as Stata
16
+ ```
17
+
18
+ ## Examples
19
+
20
+ ### Write an SPSS file
21
+
22
+ ```python
23
+ import pyarrow as pa
24
+ import readstat_arrow
25
+ from readstat_arrow import Metadata
26
+
27
+ table = pa.table({"id": pa.array([1, 2, 3], pa.int32()), "sex": pa.array([1, 2, None])})
28
+
29
+ meta = Metadata(
30
+ file_label="Tiny survey",
31
+ variable_labels={"id": "Respondent id", "sex": "Sex"},
32
+ value_labels={
33
+ "sex": [
34
+ {"value": 1, "label": "Male"},
35
+ {"value": 2, "label": "Female"},
36
+ {"value": 9, "label": "Unknown"},
37
+ ]
38
+ },
39
+ formats={"sex": "F1.0"},
40
+ measures={"sex": "nominal"},
41
+ storage_widths={"sex": 1},
42
+ missing_values={"sex": {"values": [9]}}, # or a range: {"lo": 90, "hi": 99}
43
+ )
44
+
45
+ readstat_arrow.write_sav("tiny.sav", table, meta)
46
+
47
+ readstat_arrow.write_sav("bare.sav", table) # no metadata at all: just the columns
48
+ ```
49
+
50
+ `Metadata` is a set of mappings from variable name to one attribute — `variable_labels`,
51
+ `value_labels`, `formats`, `storage_widths`, `display_widths`, `measures`, `alignments`,
52
+ `missing_values` — plus the file-level `file_label`, `notes` and `multiple_response_sets`. Which
53
+ columns exist, and in what order, is the Arrow schema's business. Nothing is required: a name absent
54
+ from a mapping declares nothing and the writer falls back to the format's own default, and the
55
+ metadata may be left out altogether. SPSS stores every number as a double, so `id` reads back as
56
+ `float64`, not `int32`.
57
+
58
+ ### Write a Stata file, whole or in batches
59
+
60
+ ```python
61
+ import readstat_arrow
62
+
63
+ table, meta = readstat_arrow.read_sav("survey.sav")
64
+ readstat_arrow.write_dta("survey.dta", table, meta) # a whole table in one call
65
+
66
+ # or stream it, without holding every row in memory
67
+ with readstat_arrow.DtaWriter("survey.dta", table.schema, table.num_rows, meta) as writer:
68
+ for batch in table.to_batches(max_chunksize=100_000):
69
+ writer.write_batch(batch)
70
+ ```
71
+
72
+ `DtaWriter` (and `SavWriter`) needs the final row count up front.
73
+
74
+ ### Read an SPSS file
75
+
76
+ ```python
77
+ import readstat_arrow
78
+
79
+ table, meta = readstat_arrow.read_sav("survey.sav")
80
+
81
+ table.num_rows # -> 12_345
82
+ table.schema # dates and times are already temporal types
83
+ table.column("q1")
84
+
85
+ table.column_names # -> every variable, in file order
86
+ meta.file_label # -> "2026 satisfaction survey"
87
+ meta.variable_labels["q1"] # -> "How satisfied are you ...?"
88
+ meta.value_labels["q1"] # -> [{"value": 1, "label": "Very unsatisfied"}, ...]
89
+ meta.missing_values["q1"] # -> {"values": [9]}, or {"lo": 90, "hi": 99}
90
+
91
+ table.to_pandas() # or
92
+ import polars as pl
93
+
94
+ pl.from_arrow(table)
95
+ ```
96
+
97
+ ### Read part of a Stata file
98
+
99
+ ```python
100
+ import readstat_arrow
101
+
102
+ # header only, no rows decoded: the schema a full read would give comes back
103
+ # beside the metadata
104
+ row_count, schema, meta = readstat_arrow.read_dta_metadata("panel.dta")
105
+ schema.names # -> ["id", "income", ...], the variables in file order
106
+ schema.field("income").type # -> the type read_dta would give that column
107
+
108
+ table, meta = readstat_arrow.read_dta(
109
+ "panel.dta",
110
+ columns=["id", "income"], # only these, in this order
111
+ row_offset=1_000, # skip the first 1_000 rows
112
+ row_limit=1_000, # then read at most 1_000
113
+ preserve_user_missing=True, # keep Stata's tagged missings .a-.z
114
+ )
115
+
116
+ # with preserve_user_missing every numeric column is struct<value, tag>
117
+ table.column("income")[0].as_py() # -> {"value": None, "tag": "a"}, Stata's .a
118
+ table.column("income")[1].as_py() # -> {"value": 42.0, "tag": None}, a real number
119
+ table.column("income")[2].as_py() # -> None, a plain .
120
+ ```
121
+
122
+ ### Read a big file in less memory
123
+
124
+ Where memory is the constraint, holding the whole table in it is the expensive part of reading a
125
+ file — and the type a column is stored as is often wider than its values need. A `.sav` is the worst
126
+ of it: every numeric column is a 64-bit double whatever it holds, so a survey of one-digit codes
127
+ costs 8 bytes a cell. A `.dta` has narrow types of its own — `byte`, `int`, `long`, `float` — but a
128
+ variable is only as narrow as whoever wrote the file declared it.
129
+
130
+ `scan_and_narrow_types=True` reads each column at the width its values actually need instead:
131
+
132
+ ```python
133
+ import readstat_arrow
134
+
135
+ table, meta = readstat_arrow.read_sav("big.sav", scan_and_narrow_types=True)
136
+
137
+ table.schema.field("q1").type # -> DataType(int8), where the file says double
138
+ ```
139
+
140
+ The file is parsed twice — once to measure the values, keeping none of them, then once to read them
141
+ at the widths that fit — so the trade is time for memory: about twice the wall clock of a plain read,
142
+ and no column is ever built at its stored width.
143
+
144
+ Only a type that holds the column exactly is ever chosen: integer types when every value was a whole
145
+ number, `float32` when every value round-trips through it, else `float64`. The ladder is `int8`,
146
+ `int16`, `int32`, `float32`, `float64` — there is no `int64`, which would save nothing over the
147
+ double it replaces, and no unsigned type. Strings are untouched, and a column of nothing but nulls
148
+ comes back as `int8`. Dates and times come back as the same temporal types either way: the width
149
+ they are read at only decides how much there is to convert.
150
+
151
+ ### Read from something other than a path
152
+
153
+ Every `read_*` function also takes a binary file object, so a file that arrives over the network or
154
+ out of an archive never has to be written to disk first.
155
+
156
+ ```python
157
+ import io, zipfile
158
+ import readstat_arrow
159
+
160
+ # straight out of a zip archive, without extracting it
161
+ with zipfile.ZipFile("survey.zip") as archive, archive.open("survey.sav") as member:
162
+ table, meta = readstat_arrow.read_sav(member)
163
+
164
+ # or from bytes you already have in hand
165
+ table, meta = readstat_arrow.read_sav(io.BytesIO(downloaded))
166
+
167
+ # an open file works too, and is left open where reading stopped
168
+ with open("survey.sav", "rb") as file:
169
+ row_count, schema, meta = readstat_arrow.read_sav_metadata(file)
170
+ ```
171
+
172
+ The file object must be seekable, and is read from wherever it currently is - so a `.sav` embedded
173
+ in a larger stream can be read by seeking to its first byte. The writers have taken a file object
174
+ all along.
175
+
176
+
177
+ ## Development
178
+
179
+ Requires [uv](https://docs.astral.sh/uv/) and a C compiler.
180
+
181
+ ```sh
182
+ git clone <repo-url>
183
+ cd readstat-arrow
184
+ uv sync # builds the Cython extension into .venv
185
+ uv run coverage run -m pytest && uv run coverage report || uv run coverage html
186
+ uv run ruff check . && uv run ruff format --check . && uv run mypy
187
+ ```
188
+
189
+ `uv run mypy` checks `src/` and `tests/` in strict mode. The Cython sources in
190
+ `src/readstat_arrow/_cython/` are excluded — they are typed for Cython's C type system, which mypy
191
+ cannot follow, and Cython checks them at compile time.
192
+
193
+ ReadStat is vendored as a git submodule at `vendor/ReadStat`; bump it with `git submodule update
194
+ --remote vendor/ReadStat`. `uv sync` rebuilds the extension whenever `_cython/`, `setup.py` or the
195
+ ReadStat sources change (see `[tool.uv] cache-keys` in `pyproject.toml`).
196
+
197
+ ## Layout
198
+
199
+ ```
200
+ pyproject.toml project metadata, deps, tool config (uv/ruff/mypy/pytest)
201
+ setup.py Cython extension definition (compiles ReadStat in)
202
+ vendor/ReadStat/ git submodule
203
+ src/readstat_arrow/
204
+ __init__.py public API re-exports
205
+ reader.py read_* and read_*_metadata functions, table assembly, type narrowing
206
+ writer.py SavWriter / DtaWriter, write_* functions, type planning
207
+ metadata.py the Metadata dataclass and its per-variable mappings
208
+ errors.py ReadstatError, ReadstatWarning
209
+ _formats.py FileFormat literal type
210
+ _dates.py display-format -> temporal type conversion
211
+ _cython/ everything Cython compiles (private)
212
+ parser.py pure-Python-mode Cython: ReadStat callbacks -> Arrow buffers
213
+ writer.py pure-Python-mode Cython: Arrow buffers -> readstat_insert_*
214
+ readstat.pxd C declarations for readstat.h
215
+ tests/ pytest suite; sample files under tests/data/
216
+ ```
217
+
218
+ ## Versioning
219
+
220
+ Releases use [CalVer](https://calver.org/) in the form `YYYY.MM.DD.INC0` (e.g. `2026.09.01.0`, then
221
+ `2026.09.01.1` for a fix in the same dat). There are no compatibility promises encoded in the
222
+ number. The version is set once in `pyproject.toml` and exposed as `readstat_arrow.__version__`.
223
+
224
+ ## Licence
225
+
226
+ MIT. ReadStat is MIT-licensed; the sample files under `tests/data/` come from pyreadstat (Apache
227
+ 2.0) — see `tests/data/README.md`.
@@ -0,0 +1,143 @@
1
+ [build-system]
2
+ requires = ["setuptools>=77", "cython>=3.1"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "readstat-arrow"
7
+ version = "2026.9.21.1"
8
+ description = "Read and write SPSS (.sav) and Stata (.dta) files as Apache Arrow tables via ReadStat"
9
+ readme = "README.md"
10
+ license = "MIT"
11
+ # The compiled extensions statically link the vendored ReadStat (also MIT) and,
12
+ # on Windows, win_iconv (public domain), so those notices travel with the wheels.
13
+ license-files = [
14
+ "LICENSE",
15
+ "vendor/ReadStat/LICENSE",
16
+ "vendor/win-iconv/readme.txt",
17
+ ]
18
+ requires-python = ">=3.11"
19
+ authors = [{ name = "Eirik Stavestrand" }]
20
+ keywords = ["arrow", "pyarrow", "spss", "stata", "readstat", "sav", "dta"]
21
+ classifiers = [
22
+ "Intended Audience :: Science/Research",
23
+ "Programming Language :: Python :: 3",
24
+ "Programming Language :: Cython",
25
+ "Topic :: Scientific/Engineering",
26
+ ]
27
+ dependencies = ["pyarrow>=17"]
28
+
29
+ [project.urls]
30
+ Homepage = "https://github.com/eirki/readstat-arrow"
31
+
32
+ [dependency-groups]
33
+ dev = [
34
+ "cython>=3.1",
35
+ "setuptools>=77",
36
+ "pytest>=8",
37
+ "ruff>=0.6",
38
+ "mypy>=1.11",
39
+ "coverage>=7.16.1",
40
+ "bumpver>=2026.1132",
41
+ ]
42
+
43
+ [tool.setuptools]
44
+ package-dir = { "" = "src" }
45
+ packages = ["readstat_arrow", "readstat_arrow._cython"]
46
+ include-package-data = true
47
+
48
+ [tool.setuptools.package-data]
49
+ readstat_arrow = ["py.typed"]
50
+ "readstat_arrow._cython" = ["*.pxd"]
51
+
52
+ [tool.uv]
53
+ # uv caches built wheels of the project; rebuild when any of these change.
54
+ cache-keys = [
55
+ { file = "pyproject.toml" },
56
+ { file = "setup.py" },
57
+ { file = "src/readstat_arrow/_cython/*.py" },
58
+ { file = "src/readstat_arrow/_cython/*.pxd" },
59
+ { file = "vendor/ReadStat/src/**/*.c" },
60
+ { file = "vendor/ReadStat/src/**/*.h" },
61
+ ]
62
+
63
+ [tool.cibuildwheel]
64
+ # CPython 3.11-3.14 on manylinux/musllinux (x86_64, aarch64), macOS (x86_64,
65
+ # arm64) and win_amd64. No PyPy (no pyarrow), no free-threaded builds (ditto),
66
+ # no 32-bit.
67
+ build = "cp3{11,12,13,14}-*"
68
+ skip = ["*_i686", "*-win32"]
69
+
70
+ # manylinux2014 is end-of-life and pyarrow targets manylinux_2_28; match it.
71
+ manylinux-x86_64-image = "manylinux_2_28"
72
+ manylinux-aarch64-image = "manylinux_2_28"
73
+
74
+ # Run the real suite against the installed wheel on every target. This is the
75
+ # only place the vendored win_iconv and musl's iconv get exercised: tests/data
76
+ # includes Hebrew and Telugu .sav files, so a charset conversion that differs
77
+ # from glibc's iconv surfaces here as a red build rather than as user mojibake.
78
+ test-requires = ["pytest"]
79
+ test-sources = ["tests"]
80
+ test-command = "pytest tests"
81
+
82
+ [tool.pytest.ini_options]
83
+ testpaths = ["tests"]
84
+ addopts = "-ra"
85
+
86
+ [tool.coverage.run]
87
+ source = ["src/readstat_arrow"]
88
+
89
+ [tool.coverage.report]
90
+ show_missing = true
91
+ skip_covered = true
92
+ fail_under = 100
93
+ exclude_also = ["pragma: no cover", ".*assert_never.*"]
94
+ omit = ["src/readstat_arrow/_cython/*"]
95
+
96
+
97
+ [tool.ruff]
98
+ line-length = 110
99
+ target-version = "py311"
100
+ src = ["src", "tests"]
101
+ extend-exclude = ["*.md"] # keep hand-aligned comments in README code blocks
102
+
103
+ [tool.ruff.lint]
104
+ select = ["E", "F", "I", "UP", "B", "SIM", "RUF"]
105
+ # _cython/ is Cython pure-Python mode; it legitimately uses patterns ruff dislikes.
106
+ per-file-ignores = { "src/readstat_arrow/_cython/*.py" = [
107
+ "F401",
108
+ "F841",
109
+ "F842",
110
+ "I001",
111
+ "SIM109",
112
+ "RUF012",
113
+ "UP",
114
+ ] }
115
+ # Disable fix for unused imports (`F401`).
116
+ unfixable = ["F401"]
117
+
118
+ [tool.mypy]
119
+ python_version = "3.11"
120
+ strict = true
121
+ files = ["src/readstat_arrow", "tests"]
122
+ exclude = ["src/readstat_arrow/_cython/"]
123
+
124
+ [[tool.mypy.overrides]]
125
+ module = ["pyarrow", "pyarrow.*"]
126
+ ignore_missing_imports = true
127
+
128
+ [[tool.mypy.overrides]]
129
+ # Compiled modules: mypy would otherwise read the pure-Python-mode source and
130
+ # misjudge e.g. __cinit__ signatures. Treat them as untyped.
131
+ module = ["readstat_arrow._cython.*"]
132
+ follow_imports = "skip"
133
+
134
+ [tool.bumpver]
135
+ version_pattern = "YYYY.MM.DD.INC0"
136
+ commit = true
137
+ tag = true
138
+ push = true
139
+ current_version = "2026.9.21.1"
140
+
141
+ [tool.bumpver.file_patterns]
142
+ "pyproject.toml" = ['version = "{version}"']
143
+ "uv.lock" = ['version = "{version}"']
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+