isaac-data 0.1.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- isaac_data-0.1.3/LICENSE +21 -0
- isaac_data-0.1.3/PKG-INFO +252 -0
- isaac_data-0.1.3/README.md +209 -0
- isaac_data-0.1.3/isaac_data/__init__.py +54 -0
- isaac_data-0.1.3/isaac_data/agreement.py +397 -0
- isaac_data-0.1.3/isaac_data/cli.py +112 -0
- isaac_data-0.1.3/isaac_data/core.py +623 -0
- isaac_data-0.1.3/isaac_data/sampling.py +104 -0
- isaac_data-0.1.3/isaac_data/terms.py +31 -0
- isaac_data-0.1.3/isaac_data.egg-info/PKG-INFO +252 -0
- isaac_data-0.1.3/isaac_data.egg-info/SOURCES.txt +17 -0
- isaac_data-0.1.3/isaac_data.egg-info/dependency_links.txt +1 -0
- isaac_data-0.1.3/isaac_data.egg-info/entry_points.txt +2 -0
- isaac_data-0.1.3/isaac_data.egg-info/requires.txt +16 -0
- isaac_data-0.1.3/isaac_data.egg-info/top_level.txt +1 -0
- isaac_data-0.1.3/pyproject.toml +61 -0
- isaac_data-0.1.3/setup.cfg +4 -0
- isaac_data-0.1.3/tests/test_http_robustness.py +358 -0
- isaac_data-0.1.3/tests/test_smoke.py +85 -0
isaac_data-0.1.3/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 The Board of Trustees of the University of Illinois
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: isaac-data
|
|
3
|
+
Version: 0.1.3
|
|
4
|
+
Summary: Python loader for the ISAAC Reddit corpus (Illinois Social Attitudes Aggregate Corpus) direct-download endpoint.
|
|
5
|
+
Author-email: Babak Hemmatian <babak.hemmatian@gmail.com>
|
|
6
|
+
Maintainer: ISAAC / Social Cognition Lab, University of Illinois Urbana-Champaign
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
Project-URL: Homepage, https://isaac.psychology.illinois.edu/
|
|
9
|
+
Project-URL: Documentation, https://isaac.psychology.illinois.edu/direct-download/
|
|
10
|
+
Project-URL: Source, https://github.com/BabakHemmatian/Illinois_Social_Attitudes/tree/main/isaac-data-loader
|
|
11
|
+
Project-URL: Issues, https://github.com/BabakHemmatian/Illinois_Social_Attitudes/issues
|
|
12
|
+
Project-URL: Citation, https://github.com/BabakHemmatian/Illinois_Social_Attitudes/blob/main/CITATION.cff
|
|
13
|
+
Keywords: ISAAC,reddit,corpus,dataset,nlp,social-attitudes
|
|
14
|
+
Classifier: Development Status :: 4 - Beta
|
|
15
|
+
Classifier: Intended Audience :: Science/Research
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
23
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
24
|
+
Classifier: Topic :: Sociology
|
|
25
|
+
Classifier: Topic :: Text Processing :: Linguistic
|
|
26
|
+
Requires-Python: >=3.9
|
|
27
|
+
Description-Content-Type: text/markdown
|
|
28
|
+
License-File: LICENSE
|
|
29
|
+
Requires-Dist: requests>=2.25
|
|
30
|
+
Requires-Dist: pandas>=1.3
|
|
31
|
+
Requires-Dist: pyarrow>=10
|
|
32
|
+
Requires-Dist: fsspec>=2023.1.0
|
|
33
|
+
Requires-Dist: aiohttp>=3.8
|
|
34
|
+
Requires-Dist: platformdirs>=3
|
|
35
|
+
Provides-Extra: polars
|
|
36
|
+
Requires-Dist: polars>=0.20; extra == "polars"
|
|
37
|
+
Provides-Extra: docs
|
|
38
|
+
Requires-Dist: pdoc>=14; extra == "docs"
|
|
39
|
+
Provides-Extra: dev
|
|
40
|
+
Requires-Dist: pytest>=7; extra == "dev"
|
|
41
|
+
Requires-Dist: pdoc>=14; extra == "dev"
|
|
42
|
+
Dynamic: license-file
|
|
43
|
+
|
|
44
|
+
# isaac-data
|
|
45
|
+
|
|
46
|
+
A thin Python loader for the **ISAAC** Reddit corpus (Illinois Social Attitudes
|
|
47
|
+
Aggregate Corpus). It reads the public
|
|
48
|
+
[Direct Download catalog](https://isaac.psychology.illinois.edu/direct-download/),
|
|
49
|
+
using the published `manifest.json` as the catalog, with the data files served
|
|
50
|
+
directly from the project's public Globus collection on NCSA Taiga, so you don't
|
|
51
|
+
have to hand-build URLs or stitch months together.
|
|
52
|
+
|
|
53
|
+
- **Catalog-driven**: enumerate what exists; never hard-code filenames.
|
|
54
|
+
- **Parquet column pushdown**: ask for a few columns and only those bytes are
|
|
55
|
+
transferred over HTTP (via pyarrow + fsspec).
|
|
56
|
+
- **Resumable, cached downloads** for bulk/offline work.
|
|
57
|
+
- **pandas** out of the box.
|
|
58
|
+
|
|
59
|
+
## Install
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
pip install isaac-data
|
|
63
|
+
# or, from source:
|
|
64
|
+
pip install git+https://github.com/BabakHemmatian/Illinois_Social_Attitudes.git#subdirectory=isaac-data-loader
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
### Data Use Agreement
|
|
68
|
+
|
|
69
|
+
The first time you **access data** (`load`, `download`, or a remote `read_parquet`),
|
|
70
|
+
the package shows the ISAAC
|
|
71
|
+
[Data Use Agreement](https://github.com/BabakHemmatian/Illinois_Social_Attitudes/blob/main/Data_Use_Agreement.md)
|
|
72
|
+
and asks you to accept, then asks for your email address. Browsing the catalog
|
|
73
|
+
(`catalog`, `files`) needs no acceptance.
|
|
74
|
+
|
|
75
|
+
**What is recorded.** Acceptance is saved on your machine (in your OS config dir)
|
|
76
|
+
and sent to the ISAAC server: your email, the timestamp, and the version
|
|
77
|
+
identifiers of the agreement text you were shown. An email address is required
|
|
78
|
+
to accept.
|
|
79
|
+
|
|
80
|
+
> We ask for your email so we can notify you of changes to the Data Use
|
|
81
|
+
> Agreement and of corrections or errata affecting the corpus, and to keep a
|
|
82
|
+
> record of your acceptance. We do not share it, and we don't use it for
|
|
83
|
+
> anything else.
|
|
84
|
+
|
|
85
|
+
Sending the record is best-effort: if the server is unreachable, acceptance is
|
|
86
|
+
still recorded locally and data access proceeds.
|
|
87
|
+
|
|
88
|
+
**If the agreement changes.** The package identifies the agreement by a SHA-256
|
|
89
|
+
of its exact text and re-checks at most once a day. If the text has changed since
|
|
90
|
+
you accepted, you are shown the new version and asked to accept it again. If you
|
|
91
|
+
are offline, your existing acceptance stands.
|
|
92
|
+
|
|
93
|
+
For non-interactive use (CI, headless notebooks), accept ahead of time:
|
|
94
|
+
|
|
95
|
+
```bash
|
|
96
|
+
isaac-data accept-agreement # interactive review + accept
|
|
97
|
+
isaac-data accept-agreement --yes --email you@x.edu # accept non-interactively
|
|
98
|
+
isaac-data accept-agreement --status # show / --withdraw to revoke
|
|
99
|
+
```
|
|
100
|
+
…or set `ISAAC_ACCEPT_AGREEMENT=1` **together with** `ISAAC_AGREEMENT_EMAIL`:
|
|
101
|
+
there is no prompt to fall back on in a non-interactive session, so opting in
|
|
102
|
+
without an address raises `AgreementNotAccepted` rather than recording an
|
|
103
|
+
anonymous acceptance.
|
|
104
|
+
|
|
105
|
+
> **Renamed in 0.1.2.** This document was previously the "Terms of Use". The old
|
|
106
|
+
> names still work: `isaac-data accept-terms`, `ISAAC_ACCEPT_TERMS`,
|
|
107
|
+
> `TermsNotAccepted`, `isaac_data.accept_terms`, and `import isaac_data.terms`.
|
|
108
|
+
> Existing local acceptance records remain valid; no need to re-accept.
|
|
109
|
+
|
|
110
|
+
## Quick start
|
|
111
|
+
|
|
112
|
+
```python
|
|
113
|
+
import isaac_data as isaac
|
|
114
|
+
|
|
115
|
+
# 1) What's available?
|
|
116
|
+
cat = isaac.catalog() # full manifest as a DataFrame
|
|
117
|
+
race = isaac.files("race", "2018-01", "2018-12") # filter by category + months
|
|
118
|
+
|
|
119
|
+
# 2) Load a slice: only the columns you need (pushed down over HTTP)
|
|
120
|
+
df = isaac.load("race", "2018-03", "2018-03", columns=["text", "score"])
|
|
121
|
+
|
|
122
|
+
# 3) Stratified sample: 1000 rows TOTAL, spread equally across the 12 months
|
|
123
|
+
# (uniform within each month, reproducible). Pass columns= when sampling.
|
|
124
|
+
sample = isaac.load("age", "2015-01", "2015-12", columns=["text"], n=1000, seed=0)
|
|
125
|
+
|
|
126
|
+
# 4) Bulk download for offline use (resumable, cached)
|
|
127
|
+
paths = isaac.download("weight", "2020-01", "2020-12", dest="./weight2020")
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
Categories: `ability, age, race, sexuality, skin_tone, weight`
|
|
131
|
+
(monthly, 2007-01 → 2023-12). Both `parquet` (default, recommended) and `csv`.
|
|
132
|
+
|
|
133
|
+
## How it works
|
|
134
|
+
|
|
135
|
+
The package is three layers: **discover → read/fetch → configure**:
|
|
136
|
+
|
|
137
|
+
1. **Discover.** `catalog()` downloads the published `manifest.json` (the
|
|
138
|
+
authoritative list of every file) and returns it as a DataFrame, cached for
|
|
139
|
+
24h. `files(...)` filters that catalog by category, month range, and format.
|
|
140
|
+
Neither transfers any corpus data, so you can inspect sizes and row-counts
|
|
141
|
+
before pulling anything.
|
|
142
|
+
2. **Read or fetch.**
|
|
143
|
+
- `load(...)` is the main entry point. It selects files, then for **parquet**
|
|
144
|
+
streams *only the columns you ask for* over HTTP: it reads the file footer,
|
|
145
|
+
then just those column chunks, so `columns=["text","score"]` from a 285 MB
|
|
146
|
+
file moves a few MB, not 285. `n=` draws a **stratified total**, spread
|
|
147
|
+
equally across the selected months, uniform within each (matching the web
|
|
148
|
+
app), reading only the selected columns of the row groups that contain
|
|
149
|
+
sampled rows. Each row is tagged with `_category`/`_month`; a `max_bytes`
|
|
150
|
+
guard prevents accidental hundred-GB full loads.
|
|
151
|
+
- `read_parquet(url, columns=...)` is the single-file primitive `load` uses.
|
|
152
|
+
- `download(...)` fetches whole files to disk (resumable, skips complete ones)
|
|
153
|
+
without loading them into memory, for offline work or other tools
|
|
154
|
+
(DuckDB, Spark).
|
|
155
|
+
3. **Configure.** Reads and downloads are cached under an OS-native directory
|
|
156
|
+
(`cache_dir()` / `set_cache_dir()` / `$ISAAC_DATA_CACHE`), and the first data
|
|
157
|
+
access prompts for Data-Use-Agreement acceptance (recorded locally, and on
|
|
158
|
+
the ISAAC server when you provide an email).
|
|
159
|
+
|
|
160
|
+
In short: *catalog tells you what exists → files narrows it → load streams just
|
|
161
|
+
the columns you need (or download grabs whole files) → the cache avoids repeat
|
|
162
|
+
transfers.*
|
|
163
|
+
|
|
164
|
+
> Full per-argument reference lives in the function docstrings (`help(isaac.load)`,
|
|
165
|
+
> IDE tooltips) and the generated [API docs](#documentation); the table below is
|
|
166
|
+
> a summary.
|
|
167
|
+
|
|
168
|
+
## API
|
|
169
|
+
|
|
170
|
+
| Function | Purpose |
|
|
171
|
+
|---|---|
|
|
172
|
+
| `catalog(refresh=False)` | Full manifest as a DataFrame (cached 24h). |
|
|
173
|
+
| `files(category, start, end, fmt="parquet")` | Filtered file list. |
|
|
174
|
+
| `load(..., columns=None, n=None, seed=None, combine=True, cache=False)` | Read into pandas; column pushdown for parquet; `n` samples rows per file. |
|
|
175
|
+
| `read_parquet(url, columns=None)` | Read one parquet file (local or http) into pandas. |
|
|
176
|
+
| `download(..., dest=None)` | Resumable, cached bulk download; returns local paths. |
|
|
177
|
+
| `set_cache_dir(path)` / `cache_dir()` | Manage the local cache (default `~/.cache/isaac-data`, or `$ISAAC_DATA_CACHE`). |
|
|
178
|
+
|
|
179
|
+
`load()` has a safety guardrail (`max_bytes`, default 5 GB): it refuses very
|
|
180
|
+
large selections unless you pass `columns=`, set `n=`, raise `max_bytes=`, or use
|
|
181
|
+
`download()`.
|
|
182
|
+
|
|
183
|
+
## CLI
|
|
184
|
+
|
|
185
|
+
```bash
|
|
186
|
+
isaac-data info
|
|
187
|
+
isaac-data ls --category race --start 2018-01 --end 2018-12
|
|
188
|
+
isaac-data download --category age --start 2015-01 --end 2015-12 --dest ./age2015
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
## Documentation
|
|
192
|
+
|
|
193
|
+
The full per-argument API reference is generated from the docstrings with
|
|
194
|
+
[pdoc](https://pdoc.dev):
|
|
195
|
+
|
|
196
|
+
```bash
|
|
197
|
+
pip install "isaac-data[docs]"
|
|
198
|
+
pdoc -d google isaac_data -o docs/api # static HTML into docs/api/
|
|
199
|
+
pdoc -d google isaac_data # or a live preview server
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
(`docs/api/` is git-ignored; publish it to GitHub Pages if you want a hosted reference.)
|
|
203
|
+
|
|
204
|
+
## Requirements & platform notes
|
|
205
|
+
|
|
206
|
+
- **Python 3.9+**, on **Windows, macOS (Intel & Apple Silicon), and Linux**. All
|
|
207
|
+
dependencies ship prebuilt wheels for these platforms.
|
|
208
|
+
- **Install footprint**: `pyarrow` is a large dependency (~100 MB installed) and
|
|
209
|
+
`aiohttp` is a compiled extension. Fine on a laptop; size-conscious in slim
|
|
210
|
+
CI/container images.
|
|
211
|
+
- **Cache location** is OS-native (via `platformdirs`): `~/Library/Caches/isaac-data`
|
|
212
|
+
(macOS), `%LOCALAPPDATA%\isaac-data\Cache` (Windows), `~/.cache/isaac-data`
|
|
213
|
+
(Linux). Override with `isaac.set_cache_dir(...)` or `$ISAAC_DATA_CACHE`.
|
|
214
|
+
|
|
215
|
+
## Citation
|
|
216
|
+
|
|
217
|
+
Please cite the ISAAC paper. **One citation covers the whole project**: the
|
|
218
|
+
corpus, the pipeline, the models, and this package. Please do not cite the
|
|
219
|
+
package separately.
|
|
220
|
+
|
|
221
|
+
```bibtex
|
|
222
|
+
@article{hemmatian2026isaac,
|
|
223
|
+
author = {Hemmatian, Babak and Hadjarab, Sarah and Chen, Jessica and Kurdi, Benedek},
|
|
224
|
+
title = {The {Illinois} Social Attitudes Aggregate Corpus ({ISAAC}): An Open Tool and Reproducible Pipeline for Analyzing Social Group Discourse at Scale},
|
|
225
|
+
year = {2026},
|
|
226
|
+
note = {Manuscript submitted for publication}
|
|
227
|
+
}
|
|
228
|
+
```
|
|
229
|
+
|
|
230
|
+
## License
|
|
231
|
+
|
|
232
|
+
This package is released under the [MIT License](./LICENSE). The ISAAC corpus it
|
|
233
|
+
downloads is governed separately by the
|
|
234
|
+
[Data Use Agreement](https://github.com/BabakHemmatian/Illinois_Social_Attitudes/blob/main/Data_Use_Agreement.md),
|
|
235
|
+
which the package presents for acceptance before any data access.
|
|
236
|
+
|
|
237
|
+
## Notes
|
|
238
|
+
|
|
239
|
+
- **Parquet is recommended** for scripting (column projection, smaller transfers).
|
|
240
|
+
The final labeled ISAAC release will add many per-post fields (moralization,
|
|
241
|
+
sentiment, generalization, emotion, location), and column pushdown makes those
|
|
242
|
+
cheap to query.
|
|
243
|
+
- For SQL-style predicate pushdown without Python, query the parquet directly
|
|
244
|
+
with DuckDB (reads only the columns/row groups your query needs):
|
|
245
|
+
```sql
|
|
246
|
+
SELECT author, score FROM
|
|
247
|
+
read_parquet('https://isaac.psychology.illinois.edu/data/race/RC_2018-03.parquet')
|
|
248
|
+
WHERE score > 100;
|
|
249
|
+
```
|
|
250
|
+
HTTP has no directory listing, so wildcard globs don't work; for multiple
|
|
251
|
+
months pass an explicit URL list, e.g. from the loader:
|
|
252
|
+
`duckdb.sql("... read_parquet($u) ...", params={"u": isaac.files("race","2018-01","2018-12").url.tolist()})`.
|
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
# isaac-data
|
|
2
|
+
|
|
3
|
+
A thin Python loader for the **ISAAC** Reddit corpus (Illinois Social Attitudes
|
|
4
|
+
Aggregate Corpus). It reads the public
|
|
5
|
+
[Direct Download catalog](https://isaac.psychology.illinois.edu/direct-download/),
|
|
6
|
+
using the published `manifest.json` as the catalog, with the data files served
|
|
7
|
+
directly from the project's public Globus collection on NCSA Taiga, so you don't
|
|
8
|
+
have to hand-build URLs or stitch months together.
|
|
9
|
+
|
|
10
|
+
- **Catalog-driven**: enumerate what exists; never hard-code filenames.
|
|
11
|
+
- **Parquet column pushdown**: ask for a few columns and only those bytes are
|
|
12
|
+
transferred over HTTP (via pyarrow + fsspec).
|
|
13
|
+
- **Resumable, cached downloads** for bulk/offline work.
|
|
14
|
+
- **pandas** out of the box.
|
|
15
|
+
|
|
16
|
+
## Install
|
|
17
|
+
|
|
18
|
+
```bash
|
|
19
|
+
pip install isaac-data
|
|
20
|
+
# or, from source:
|
|
21
|
+
pip install git+https://github.com/BabakHemmatian/Illinois_Social_Attitudes.git#subdirectory=isaac-data-loader
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
### Data Use Agreement
|
|
25
|
+
|
|
26
|
+
The first time you **access data** (`load`, `download`, or a remote `read_parquet`),
|
|
27
|
+
the package shows the ISAAC
|
|
28
|
+
[Data Use Agreement](https://github.com/BabakHemmatian/Illinois_Social_Attitudes/blob/main/Data_Use_Agreement.md)
|
|
29
|
+
and asks you to accept, then asks for your email address. Browsing the catalog
|
|
30
|
+
(`catalog`, `files`) needs no acceptance.
|
|
31
|
+
|
|
32
|
+
**What is recorded.** Acceptance is saved on your machine (in your OS config dir)
|
|
33
|
+
and sent to the ISAAC server: your email, the timestamp, and the version
|
|
34
|
+
identifiers of the agreement text you were shown. An email address is required
|
|
35
|
+
to accept.
|
|
36
|
+
|
|
37
|
+
> We ask for your email so we can notify you of changes to the Data Use
|
|
38
|
+
> Agreement and of corrections or errata affecting the corpus, and to keep a
|
|
39
|
+
> record of your acceptance. We do not share it, and we don't use it for
|
|
40
|
+
> anything else.
|
|
41
|
+
|
|
42
|
+
Sending the record is best-effort: if the server is unreachable, acceptance is
|
|
43
|
+
still recorded locally and data access proceeds.
|
|
44
|
+
|
|
45
|
+
**If the agreement changes.** The package identifies the agreement by a SHA-256
|
|
46
|
+
of its exact text and re-checks at most once a day. If the text has changed since
|
|
47
|
+
you accepted, you are shown the new version and asked to accept it again. If you
|
|
48
|
+
are offline, your existing acceptance stands.
|
|
49
|
+
|
|
50
|
+
For non-interactive use (CI, headless notebooks), accept ahead of time:
|
|
51
|
+
|
|
52
|
+
```bash
|
|
53
|
+
isaac-data accept-agreement # interactive review + accept
|
|
54
|
+
isaac-data accept-agreement --yes --email you@x.edu # accept non-interactively
|
|
55
|
+
isaac-data accept-agreement --status # show / --withdraw to revoke
|
|
56
|
+
```
|
|
57
|
+
…or set `ISAAC_ACCEPT_AGREEMENT=1` **together with** `ISAAC_AGREEMENT_EMAIL`:
|
|
58
|
+
there is no prompt to fall back on in a non-interactive session, so opting in
|
|
59
|
+
without an address raises `AgreementNotAccepted` rather than recording an
|
|
60
|
+
anonymous acceptance.
|
|
61
|
+
|
|
62
|
+
> **Renamed in 0.1.2.** This document was previously the "Terms of Use". The old
|
|
63
|
+
> names still work: `isaac-data accept-terms`, `ISAAC_ACCEPT_TERMS`,
|
|
64
|
+
> `TermsNotAccepted`, `isaac_data.accept_terms`, and `import isaac_data.terms`.
|
|
65
|
+
> Existing local acceptance records remain valid; no need to re-accept.
|
|
66
|
+
|
|
67
|
+
## Quick start
|
|
68
|
+
|
|
69
|
+
```python
|
|
70
|
+
import isaac_data as isaac
|
|
71
|
+
|
|
72
|
+
# 1) What's available?
|
|
73
|
+
cat = isaac.catalog() # full manifest as a DataFrame
|
|
74
|
+
race = isaac.files("race", "2018-01", "2018-12") # filter by category + months
|
|
75
|
+
|
|
76
|
+
# 2) Load a slice: only the columns you need (pushed down over HTTP)
|
|
77
|
+
df = isaac.load("race", "2018-03", "2018-03", columns=["text", "score"])
|
|
78
|
+
|
|
79
|
+
# 3) Stratified sample: 1000 rows TOTAL, spread equally across the 12 months
|
|
80
|
+
# (uniform within each month, reproducible). Pass columns= when sampling.
|
|
81
|
+
sample = isaac.load("age", "2015-01", "2015-12", columns=["text"], n=1000, seed=0)
|
|
82
|
+
|
|
83
|
+
# 4) Bulk download for offline use (resumable, cached)
|
|
84
|
+
paths = isaac.download("weight", "2020-01", "2020-12", dest="./weight2020")
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
Categories: `ability, age, race, sexuality, skin_tone, weight`
|
|
88
|
+
(monthly, 2007-01 → 2023-12). Both `parquet` (default, recommended) and `csv`.
|
|
89
|
+
|
|
90
|
+
## How it works
|
|
91
|
+
|
|
92
|
+
The package is three layers: **discover → read/fetch → configure**:
|
|
93
|
+
|
|
94
|
+
1. **Discover.** `catalog()` downloads the published `manifest.json` (the
|
|
95
|
+
authoritative list of every file) and returns it as a DataFrame, cached for
|
|
96
|
+
24h. `files(...)` filters that catalog by category, month range, and format.
|
|
97
|
+
Neither transfers any corpus data, so you can inspect sizes and row-counts
|
|
98
|
+
before pulling anything.
|
|
99
|
+
2. **Read or fetch.**
|
|
100
|
+
- `load(...)` is the main entry point. It selects files, then for **parquet**
|
|
101
|
+
streams *only the columns you ask for* over HTTP: it reads the file footer,
|
|
102
|
+
then just those column chunks, so `columns=["text","score"]` from a 285 MB
|
|
103
|
+
file moves a few MB, not 285. `n=` draws a **stratified total**, spread
|
|
104
|
+
equally across the selected months, uniform within each (matching the web
|
|
105
|
+
app), reading only the selected columns of the row groups that contain
|
|
106
|
+
sampled rows. Each row is tagged with `_category`/`_month`; a `max_bytes`
|
|
107
|
+
guard prevents accidental hundred-GB full loads.
|
|
108
|
+
- `read_parquet(url, columns=...)` is the single-file primitive `load` uses.
|
|
109
|
+
- `download(...)` fetches whole files to disk (resumable, skips complete ones)
|
|
110
|
+
without loading them into memory, for offline work or other tools
|
|
111
|
+
(DuckDB, Spark).
|
|
112
|
+
3. **Configure.** Reads and downloads are cached under an OS-native directory
|
|
113
|
+
(`cache_dir()` / `set_cache_dir()` / `$ISAAC_DATA_CACHE`), and the first data
|
|
114
|
+
access prompts for Data-Use-Agreement acceptance (recorded locally, and on
|
|
115
|
+
the ISAAC server when you provide an email).
|
|
116
|
+
|
|
117
|
+
In short: *catalog tells you what exists → files narrows it → load streams just
|
|
118
|
+
the columns you need (or download grabs whole files) → the cache avoids repeat
|
|
119
|
+
transfers.*
|
|
120
|
+
|
|
121
|
+
> Full per-argument reference lives in the function docstrings (`help(isaac.load)`,
|
|
122
|
+
> IDE tooltips) and the generated [API docs](#documentation); the table below is
|
|
123
|
+
> a summary.
|
|
124
|
+
|
|
125
|
+
## API
|
|
126
|
+
|
|
127
|
+
| Function | Purpose |
|
|
128
|
+
|---|---|
|
|
129
|
+
| `catalog(refresh=False)` | Full manifest as a DataFrame (cached 24h). |
|
|
130
|
+
| `files(category, start, end, fmt="parquet")` | Filtered file list. |
|
|
131
|
+
| `load(..., columns=None, n=None, seed=None, combine=True, cache=False)` | Read into pandas; column pushdown for parquet; `n` samples rows per file. |
|
|
132
|
+
| `read_parquet(url, columns=None)` | Read one parquet file (local or http) into pandas. |
|
|
133
|
+
| `download(..., dest=None)` | Resumable, cached bulk download; returns local paths. |
|
|
134
|
+
| `set_cache_dir(path)` / `cache_dir()` | Manage the local cache (default `~/.cache/isaac-data`, or `$ISAAC_DATA_CACHE`). |
|
|
135
|
+
|
|
136
|
+
`load()` has a safety guardrail (`max_bytes`, default 5 GB): it refuses very
|
|
137
|
+
large selections unless you pass `columns=`, set `n=`, raise `max_bytes=`, or use
|
|
138
|
+
`download()`.
|
|
139
|
+
|
|
140
|
+
## CLI
|
|
141
|
+
|
|
142
|
+
```bash
|
|
143
|
+
isaac-data info
|
|
144
|
+
isaac-data ls --category race --start 2018-01 --end 2018-12
|
|
145
|
+
isaac-data download --category age --start 2015-01 --end 2015-12 --dest ./age2015
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
## Documentation
|
|
149
|
+
|
|
150
|
+
The full per-argument API reference is generated from the docstrings with
|
|
151
|
+
[pdoc](https://pdoc.dev):
|
|
152
|
+
|
|
153
|
+
```bash
|
|
154
|
+
pip install "isaac-data[docs]"
|
|
155
|
+
pdoc -d google isaac_data -o docs/api # static HTML into docs/api/
|
|
156
|
+
pdoc -d google isaac_data # or a live preview server
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
(`docs/api/` is git-ignored; publish it to GitHub Pages if you want a hosted reference.)
|
|
160
|
+
|
|
161
|
+
## Requirements & platform notes
|
|
162
|
+
|
|
163
|
+
- **Python 3.9+**, on **Windows, macOS (Intel & Apple Silicon), and Linux**. All
|
|
164
|
+
dependencies ship prebuilt wheels for these platforms.
|
|
165
|
+
- **Install footprint**: `pyarrow` is a large dependency (~100 MB installed) and
|
|
166
|
+
`aiohttp` is a compiled extension. Fine on a laptop; size-conscious in slim
|
|
167
|
+
CI/container images.
|
|
168
|
+
- **Cache location** is OS-native (via `platformdirs`): `~/Library/Caches/isaac-data`
|
|
169
|
+
(macOS), `%LOCALAPPDATA%\isaac-data\Cache` (Windows), `~/.cache/isaac-data`
|
|
170
|
+
(Linux). Override with `isaac.set_cache_dir(...)` or `$ISAAC_DATA_CACHE`.
|
|
171
|
+
|
|
172
|
+
## Citation
|
|
173
|
+
|
|
174
|
+
Please cite the ISAAC paper. **One citation covers the whole project**: the
|
|
175
|
+
corpus, the pipeline, the models, and this package. Please do not cite the
|
|
176
|
+
package separately.
|
|
177
|
+
|
|
178
|
+
```bibtex
|
|
179
|
+
@article{hemmatian2026isaac,
|
|
180
|
+
author = {Hemmatian, Babak and Hadjarab, Sarah and Chen, Jessica and Kurdi, Benedek},
|
|
181
|
+
title = {The {Illinois} Social Attitudes Aggregate Corpus ({ISAAC}): An Open Tool and Reproducible Pipeline for Analyzing Social Group Discourse at Scale},
|
|
182
|
+
year = {2026},
|
|
183
|
+
note = {Manuscript submitted for publication}
|
|
184
|
+
}
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
## License
|
|
188
|
+
|
|
189
|
+
This package is released under the [MIT License](./LICENSE). The ISAAC corpus it
|
|
190
|
+
downloads is governed separately by the
|
|
191
|
+
[Data Use Agreement](https://github.com/BabakHemmatian/Illinois_Social_Attitudes/blob/main/Data_Use_Agreement.md),
|
|
192
|
+
which the package presents for acceptance before any data access.
|
|
193
|
+
|
|
194
|
+
## Notes
|
|
195
|
+
|
|
196
|
+
- **Parquet is recommended** for scripting (column projection, smaller transfers).
|
|
197
|
+
The final labeled ISAAC release will add many per-post fields (moralization,
|
|
198
|
+
sentiment, generalization, emotion, location), and column pushdown makes those
|
|
199
|
+
cheap to query.
|
|
200
|
+
- For SQL-style predicate pushdown without Python, query the parquet directly
|
|
201
|
+
with DuckDB (reads only the columns/row groups your query needs):
|
|
202
|
+
```sql
|
|
203
|
+
SELECT author, score FROM
|
|
204
|
+
read_parquet('https://isaac.psychology.illinois.edu/data/race/RC_2018-03.parquet')
|
|
205
|
+
WHERE score > 100;
|
|
206
|
+
```
|
|
207
|
+
HTTP has no directory listing, so wildcard globs don't work; for multiple
|
|
208
|
+
months pass an explicit URL list, e.g. from the loader:
|
|
209
|
+
`duckdb.sql("... read_parquet($u) ...", params={"u": isaac.files("race","2018-01","2018-12").url.tolist()})`.
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""isaac-data: a thin Python loader for the ISAAC Reddit corpus.
|
|
2
|
+
|
|
3
|
+
Reads the ISAAC Direct Download endpoint
|
|
4
|
+
(https://isaac.psychology.illinois.edu/direct-download/) using the published
|
|
5
|
+
manifest as a catalog. Parquet reads support column pushdown over HTTP.
|
|
6
|
+
|
|
7
|
+
Quick start
|
|
8
|
+
-----------
|
|
9
|
+
>>> import isaac_data as isaac
|
|
10
|
+
>>> isaac.files("race", "2018-01", "2018-12") # what's available
|
|
11
|
+
>>> df = isaac.load("race", "2018-03", "2018-03", # one month, two columns
|
|
12
|
+
... columns=["text", "score"])
|
|
13
|
+
>>> isaac.download("age", "2015-01", "2015-12", dest="./age2015") # bulk fetch
|
|
14
|
+
"""
|
|
15
|
+
from .core import (
|
|
16
|
+
BASE_URL,
|
|
17
|
+
CATEGORIES,
|
|
18
|
+
DATA_BASE,
|
|
19
|
+
MANIFEST_URL,
|
|
20
|
+
DataHostUnavailable,
|
|
21
|
+
cache_dir,
|
|
22
|
+
catalog,
|
|
23
|
+
download,
|
|
24
|
+
files,
|
|
25
|
+
load,
|
|
26
|
+
read_parquet,
|
|
27
|
+
set_cache_dir,
|
|
28
|
+
)
|
|
29
|
+
from .agreement import (
|
|
30
|
+
AgreementNotAccepted,
|
|
31
|
+
accept_agreement,
|
|
32
|
+
is_accepted,
|
|
33
|
+
status as agreement_status,
|
|
34
|
+
withdraw as withdraw_agreement,
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
# Pre-2026-07-25 names ("Terms of Use" era), kept importable for compatibility.
|
|
38
|
+
TermsNotAccepted = AgreementNotAccepted
|
|
39
|
+
accept_terms = accept_agreement
|
|
40
|
+
terms_status = agreement_status
|
|
41
|
+
withdraw_terms = withdraw_agreement
|
|
42
|
+
|
|
43
|
+
__version__ = "0.1.3"
|
|
44
|
+
|
|
45
|
+
__all__ = [
|
|
46
|
+
"__version__",
|
|
47
|
+
"BASE_URL", "DATA_BASE", "MANIFEST_URL", "CATEGORIES",
|
|
48
|
+
"cache_dir", "set_cache_dir", "catalog", "files",
|
|
49
|
+
"download", "read_parquet", "load", "DataHostUnavailable",
|
|
50
|
+
"accept_agreement", "is_accepted", "agreement_status", "withdraw_agreement",
|
|
51
|
+
"AgreementNotAccepted",
|
|
52
|
+
# deprecated aliases
|
|
53
|
+
"accept_terms", "terms_status", "withdraw_terms", "TermsNotAccepted",
|
|
54
|
+
]
|