isaac-data 0.1.3__py3-none-any.whl

isaac_data/__init__.py

@@ -0,0 +1,54 @@
+"""isaac-data: a thin Python loader for the ISAAC Reddit corpus.
+
+Reads the ISAAC Direct Download endpoint
+(https://isaac.psychology.illinois.edu/direct-download/) using the published
+manifest as a catalog. Parquet reads support column pushdown over HTTP.
+
+Quick start
+-----------
+>>> import isaac_data as isaac
+>>> isaac.files("race", "2018-01", "2018-12")          # what's available
+>>> df = isaac.load("race", "2018-03", "2018-03",       # one month, two columns
+...                 columns=["text", "score"])
+>>> isaac.download("age", "2015-01", "2015-12", dest="./age2015")  # bulk fetch
+"""
+from .core import (
+    BASE_URL,
+    CATEGORIES,
+    DATA_BASE,
+    MANIFEST_URL,
+    DataHostUnavailable,
+    cache_dir,
+    catalog,
+    download,
+    files,
+    load,
+    read_parquet,
+    set_cache_dir,
+)
+from .agreement import (
+    AgreementNotAccepted,
+    accept_agreement,
+    is_accepted,
+    status as agreement_status,
+    withdraw as withdraw_agreement,
+)
+
+# Pre-2026-07-25 names ("Terms of Use" era), kept importable for compatibility.
+TermsNotAccepted = AgreementNotAccepted
+accept_terms = accept_agreement
+terms_status = agreement_status
+withdraw_terms = withdraw_agreement
+
+__version__ = "0.1.3"
+
+__all__ = [
+    "__version__",
+    "BASE_URL", "DATA_BASE", "MANIFEST_URL", "CATEGORIES",
+    "cache_dir", "set_cache_dir", "catalog", "files",
+    "download", "read_parquet", "load", "DataHostUnavailable",
+    "accept_agreement", "is_accepted", "agreement_status", "withdraw_agreement",
+    "AgreementNotAccepted",
+    # deprecated aliases
+    "accept_terms", "terms_status", "withdraw_terms", "TermsNotAccepted",
+]

isaac_data/agreement.py

@@ -0,0 +1,397 @@
+"""First-run Data Use Agreement acceptance.
+
+Data access (`load`, `download`, remote `read_parquet`) requires acceptance of
+the ISAAC Data Use Agreement. Browsing the catalog (`catalog`, `files`) does not.
+
+Accepting requires an email address. It is recorded locally (OS-native config
+dir) and posted to the ISAAC server, so the project has one record across access
+surfaces and a way to reach users about agreement changes and corpus errata. The
+POST itself is best-effort: a network failure never blocks data access, and the
+local record notes whether the server acknowledged it.
+
+The agreement text is fetched from the ISAAC server's /dua endpoint, which
+serves the live document out of the corpus repo along with a SHA-256 of the
+exact bytes. That hash is what identifies the version, so:
+
+  * the hash recorded here matches the one the website records, and
+  * if the agreement changes, `require_acceptance` notices and re-prompts
+    rather than honoring a stale acceptance forever.
+
+Non-interactive use (CI, notebooks without a TTY) must either accept beforehand
+via `isaac-data accept-agreement` or set ``ISAAC_ACCEPT_AGREEMENT=1``. Either way
+``ISAAC_AGREEMENT_EMAIL`` (or ``--email``) must supply the address, since there is
+no prompt to fall back on. Otherwise data access raises ``AgreementNotAccepted``.
+
+Renamed 2026-07-25: the document was previously called the "Terms of Use". The
+old names (module ``isaac_data.terms``, ``TermsNotAccepted``, ``accept_terms``,
+``ISAAC_ACCEPT_TERMS``, ``isaac-data accept-terms``) still work as aliases.
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import re
+import sys
+import time
+import uuid
+from pathlib import Path
+from typing import Optional
+
+AGREEMENT_PAGE = "https://github.com/BabakHemmatian/Illinois_Social_Attitudes/blob/main/Data_Use_Agreement.md"
+AGREEMENT_RAW = "https://raw.githubusercontent.com/BabakHemmatian/Illinois_Social_Attitudes/main/Data_Use_Agreement.md"
+_ENV = "ISAAC_ACCEPT_AGREEMENT"
+_ENV_LEGACY = "ISAAC_ACCEPT_TERMS"  # pre-2026-07-25 name, still honored
+_ENV_EMAIL = "ISAAC_AGREEMENT_EMAIL"
+
+# Why we ask for an email. Keep this in sync with the wording on the website and
+# the HuggingFace dataset card — and keep it accurate: do not promise uses we do
+# not actually carry out.
+EMAIL_PURPOSE = (
+    "We ask for your email so we can notify you of changes to the Data Use\n"
+    "Agreement and of corrections or errata affecting the corpus, and to keep a\n"
+    "record of your acceptance. We do not share it, and we don't use it for\n"
+    "anything else."
+)
+
+# How often to re-verify that the accepted agreement is still the current one.
+# Matches the manifest cache policy in core.py; a miss costs one small GET.
+_VERSION_CHECK_TTL_SECONDS = 24 * 3600
+
+# Deliberately permissive: this is a typo guard, not identity verification.
+_EMAIL_RE = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$")
+
+
+class AgreementNotAccepted(RuntimeError):
+    """Raised when the ISAAC Data Use Agreement has not been accepted."""
+
+
+def _env_opt_in() -> bool:
+    for name in (_ENV, _ENV_LEGACY):
+        if os.environ.get(name, "").strip().lower() in ("1", "true", "yes"):
+            return True
+    return False
+
+
+def _env_email() -> Optional[str]:
+    val = os.environ.get(_ENV_EMAIL, "").strip()
+    return val if val and _EMAIL_RE.match(val) else None
+
+
+def _base_url() -> str:
+    # Read at call time (not import time) so ISAAC_BASE_URL can be set in-process.
+    return os.environ.get("ISAAC_BASE_URL", "https://isaac.psychology.illinois.edu").rstrip("/")
+
+
+def _config_dir() -> Path:
+    override = os.environ.get("ISAAC_DATA_CONFIG")
+    if override:
+        d = Path(override).expanduser()
+    else:
+        import platformdirs
+        d = Path(platformdirs.user_config_dir("isaac-data"))
+    d.mkdir(parents=True, exist_ok=True)
+    return d
+
+
+def _record_file() -> Path:
+    return _config_dir() / "accepted.json"
+
+
+def _read_record() -> Optional[dict]:
+    f = _record_file()
+    if not f.exists():
+        return None
+    try:
+        return json.loads(f.read_text())
+    except Exception:
+        # Unreadable record: treat as "accepted, version unknown" rather than
+        # forcing a re-prompt because of a corrupt file.
+        return {"accepted": True, "record": str(f)}
+
+
+def is_accepted() -> bool:
+    """True if this machine has an acceptance record. Cheap; never hits network.
+
+    Does not check whether the accepted version is still current —
+    `require_acceptance` does that, so this stays usable as a fast predicate.
+    """
+    return _record_file().exists()
+
+
+def status() -> Optional[dict]:
+    """Return the local acceptance record, or None if not yet accepted."""
+    return _read_record()
+
+
+def fetch_agreement_record(timeout: int = 15) -> Optional[dict]:
+    """Fetch the current agreement text plus its version identifiers.
+
+    Prefers the ISAAC server's /dua endpoint so the SHA-256 recorded here is the
+    same one the website records. Falls back to raw GitHub (hashing the bytes
+    ourselves) if the server is unreachable. Returns None if both fail.
+    """
+    import requests
+    try:
+        r = requests.get(f"{_base_url()}/dua", timeout=timeout,
+                         headers={"Accept": "application/json"})
+        if r.ok:
+            d = r.json()
+            if d.get("markdown"):
+                return {
+                    "text": d["markdown"],
+                    "sha256": d.get("sha256"),
+                    "commit": d.get("commit"),
+                    "version": d.get("version"),
+                }
+    except Exception:
+        pass
+    try:
+        r = requests.get(AGREEMENT_RAW, timeout=timeout)
+        if r.ok and r.text.strip():
+            return {
+                "text": r.text,
+                "sha256": hashlib.sha256(r.content).hexdigest(),
+                "commit": None,
+                "version": None,
+            }
+    except Exception:
+        pass
+    return None
+
+
+def fetch_agreement(timeout: int = 15) -> Optional[str]:
+    """Best-effort fetch of the current Data Use Agreement text (None if unavailable)."""
+    rec = fetch_agreement_record(timeout=timeout)
+    return rec["text"] if rec else None
+
+
+def withdraw() -> bool:
+    """Delete the local acceptance record. Returns True if one existed."""
+    f = _record_file()
+    if f.exists():
+        f.unlink()
+        return True
+    return False
+
+
+def _client_id() -> str:
+    """Stable per-machine id, so repeat acceptances are recognisable as one user.
+
+    Not an identity claim — just a correlation handle for the consent log.
+    """
+    prev = _read_record() or {}
+    return prev.get("client_id") or f"pypi:{uuid.uuid4()}"
+
+
+def _post_consent(rec: dict, timeout: int = 10) -> bool:
+    """Best-effort POST of the acceptance to the ISAAC server. Never raises."""
+    if not rec.get("email"):
+        return False
+    import requests
+    from . import __version__
+    try:
+        r = requests.post(
+            f"{_base_url()}/record_consent",
+            json={
+                "uid": rec["client_id"],
+                "email": rec["email"],
+                "agreement_version": rec.get("agreement_version") or "unknown",
+                "agreement_sha256": rec.get("agreement_sha256"),
+                "agreement_commit": rec.get("agreement_commit"),
+                "accepted_at": rec["accepted_at_utc"],
+                "source": "pypi",
+                "package_version": __version__,
+            },
+            timeout=timeout,
+        )
+        return bool(r.ok)
+    except Exception:
+        return False
+
+
+def _prompt_email(out) -> str:
+    """Ask for an email address. Required; raises if one is not supplied.
+
+    An address is part of accepting the agreement (it is how we reach you about
+    changes and errata), so there is no skip option here. Ctrl-C still aborts.
+    """
+    print(EMAIL_PURPOSE, file=out)
+    print(file=out)
+    for _ in range(5):
+        try:
+            resp = input("Your email address: ").strip()
+        except EOFError:
+            break
+        if _EMAIL_RE.match(resp):
+            return resp
+        if resp:
+            print("That doesn't look like an email address; please try again.", file=out)
+        else:
+            print("An email address is required to accept the agreement.", file=out)
+    raise AgreementNotAccepted(
+        "An email address is required to accept the ISAAC Data Use Agreement; aborting."
+    )
+
+
+def _require_email(email: Optional[str]) -> str:
+    """Validate a non-interactively supplied address, with an actionable error."""
+    if email and _EMAIL_RE.match(email):
+        return email
+    raise AgreementNotAccepted(
+        "An email address is required to accept the ISAAC Data Use Agreement.\n"
+        f"Pass `isaac-data accept-agreement --email you@example.edu` or set {_ENV_EMAIL}."
+        + ("" if not email else f"\n(Got an address that doesn't parse: {email!r})")
+    )
+
+
+def _write_record(agreement: Optional[dict], email: Optional[str], via: str) -> dict:
+    from . import __version__
+    now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
+    rec = {
+        "accepted": True,
+        "accepted_at_utc": now,
+        "agreement_url": AGREEMENT_PAGE,
+        "agreement_sha256": (agreement or {}).get("sha256"),
+        "agreement_commit": (agreement or {}).get("commit"),
+        "agreement_version": (agreement or {}).get("version"),
+        "email": email,
+        "client_id": _client_id(),
+        "accepted_via": via,          # "prompt" | "env" | "assume_yes"
+        "package_version": __version__,
+        "last_version_check_utc": now,
+    }
+    rec["server_ack"] = _post_consent(rec)
+    _record_file().write_text(json.dumps(rec, indent=2) + "\n")
+    return rec
+
+
+def _print_agreement(agreement: Optional[dict], out) -> None:
+    print("\n" + "=" * 72, file=out)
+    print("ISAAC dataset: Data Use Agreement", file=out)
+    print("=" * 72, file=out)
+    if agreement and agreement.get("text"):
+        print(agreement["text"].strip(), file=out)
+        if agreement.get("version"):
+            print(f"\n[version {agreement['version']}]", file=out)
+    else:
+        print("By using the ISAAC dataset and this package you agree to the ISAAC", file=out)
+        print(f"Data Use Agreement:\n  {AGREEMENT_PAGE}", file=out)
+    print("-" * 72, file=out)
+
+
+def accept_agreement(assume_yes: bool = False, email: Optional[str] = None) -> dict:
+    """Show the Data Use Agreement and record acceptance.
+
+    assume_yes : skip the prompt (equivalent to ``ISAAC_ACCEPT_AGREEMENT=1``).
+    email      : contact address to record; falls back to ``ISAAC_AGREEMENT_EMAIL``,
+                 then to an interactive prompt. Required either way.
+
+    Raises AgreementNotAccepted if the user declines, if no email is supplied, or
+    if no terminal is available to ask on.
+    """
+    out = sys.stderr
+    agreement = fetch_agreement_record()
+    _print_agreement(agreement, out)
+
+    if assume_yes or _env_opt_in():
+        rec = _write_record(agreement, _require_email(email or _env_email()),
+                            via="assume_yes" if assume_yes else "env")
+        print(f"Agreement accepted (recorded at {_record_file()}).", file=out)
+        return rec
+
+    if not (sys.stdin and sys.stdin.isatty()):
+        raise AgreementNotAccepted(
+            "ISAAC Data Use Agreement not accepted and no interactive terminal is available.\n"
+            f"Read {AGREEMENT_PAGE}, then run `isaac-data accept-agreement` or set {_ENV}=1."
+        )
+
+    resp = input("Do you accept the ISAAC Data Use Agreement? [y/N] ").strip().lower()
+    if resp not in ("y", "yes"):
+        raise AgreementNotAccepted("ISAAC Data Use Agreement was not accepted; aborting.")
+
+    if email is None:
+        email = _env_email() or _prompt_email(out)
+
+    rec = _write_record(agreement, email, via="prompt")
+    print(f"Thank you. Acceptance recorded at {_record_file()}.", file=out)
+    return rec
+
+
+def _needs_version_recheck(rec: dict) -> bool:
+    """True if enough time has passed to re-verify the accepted version."""
+    last = rec.get("last_version_check_utc")
+    if not last:
+        return True
+    try:
+        import calendar
+        elapsed = time.time() - calendar.timegm(time.strptime(last, "%Y-%m-%dT%H:%M:%SZ"))
+    except Exception:
+        return True
+    return elapsed > _VERSION_CHECK_TTL_SECONDS
+
+
+def _touch_version_check(rec: dict) -> None:
+    rec["last_version_check_utc"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
+    try:
+        _record_file().write_text(json.dumps(rec, indent=2) + "\n")
+    except Exception:
+        pass
+
+
+def require_acceptance() -> None:
+    """Gate called before data access. Cheap once accepted and up to date.
+
+    Re-prompts if the agreement text has changed since it was accepted, checking
+    at most once per `_VERSION_CHECK_TTL_SECONDS`. If the check cannot reach the
+    network it proceeds on the existing acceptance — being offline must not
+    block a user who has already agreed.
+    """
+    rec = _read_record()
+    if rec is None:
+        if _env_opt_in():
+            _write_record(fetch_agreement_record(), _require_email(_env_email()), via="env")
+            return
+        accept_agreement()
+        return
+
+    accepted_sha = rec.get("agreement_sha256")
+    if not accepted_sha or not _needs_version_recheck(rec):
+        return
+
+    current = fetch_agreement_record()
+    if current is None or not current.get("sha256"):
+        return  # offline or server down: honor the existing acceptance
+    if current["sha256"] == accepted_sha:
+        _touch_version_check(rec)
+        return
+
+    # The agreement changed; acceptance of the old text does not carry over.
+    print("\nThe ISAAC Data Use Agreement has changed since you accepted it.",
+          file=sys.stderr)
+    if _env_opt_in():
+        # Carry the address forward from the prior acceptance where we have one.
+        _write_record(current, _require_email(rec.get("email") or _env_email()), via="env")
+        return
+    if not (sys.stdin and sys.stdin.isatty()):
+        raise AgreementNotAccepted(
+            "The ISAAC Data Use Agreement has changed and the new version has not "
+            "been accepted, and no interactive terminal is available.\n"
+            f"Review it at {AGREEMENT_PAGE}, then run `isaac-data accept-agreement` "
+            f"or set {_ENV}=1."
+        )
+    _print_agreement(current, sys.stderr)
+    resp = input("Do you accept the updated ISAAC Data Use Agreement? [y/N] ").strip().lower()
+    if resp not in ("y", "yes"):
+        raise AgreementNotAccepted(
+            "The updated ISAAC Data Use Agreement was not accepted; aborting."
+        )
+    _write_record(current, rec.get("email") or _env_email(), via="prompt")
+
+
+# ---- Pre-2026-07-25 aliases ("Terms of Use" era). Kept for compatibility. ----
+TermsNotAccepted = AgreementNotAccepted
+TERMS_PAGE = AGREEMENT_PAGE
+TERMS_RAW = AGREEMENT_RAW
+fetch_terms = fetch_agreement
+accept_terms = accept_agreement

isaac_data/cli.py

@@ -0,0 +1,112 @@
+"""Command-line interface for isaac-data.
+
+Examples
+--------
+    isaac-data ls --category race --start 2018-01 --end 2018-12
+    isaac-data info
+    isaac-data download --category age --start 2015-01 --end 2015-12 --dest ./age2015
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+
+from . import __version__
+from .core import CATEGORIES, download, files
+
+
+def _add_selection_args(p: argparse.ArgumentParser) -> None:
+    p.add_argument("-c", "--category", choices=CATEGORIES, help="social group (default: all)")
+    p.add_argument("-s", "--start", help="start month, YYYY-MM (inclusive)")
+    p.add_argument("-e", "--end", help="end month, YYYY-MM (inclusive)")
+    p.add_argument("-f", "--format", dest="fmt", default="parquet",
+                   choices=["parquet", "csv", "both"], help="file format (default: parquet)")
+
+
+def _fmt(args) -> str | None:
+    return None if args.fmt == "both" else args.fmt
+
+
+def main(argv=None) -> int:
+    parser = argparse.ArgumentParser(prog="isaac-data", description="ISAAC corpus direct-download helper.")
+    parser.add_argument("--version", action="version", version=f"isaac-data {__version__}")
+    # metavar lists only the public commands; `accept-terms` is a hidden
+    # pre-2026-07-25 alias that still parses.
+    sub = parser.add_subparsers(
+        dest="cmd", required=True,
+        metavar="{ls,info,download,accept-agreement}",
+    )
+
+    p_ls = sub.add_parser("ls", help="list matching files")
+    _add_selection_args(p_ls)
+
+    p_info = sub.add_parser("info", help="summary of the whole corpus")
+
+    p_dl = sub.add_parser("download", help="download matching files (resumable)")
+    _add_selection_args(p_dl)
+    p_dl.add_argument("-d", "--dest", help="destination directory (default: cache)")
+
+    for _name, _kw in (
+        ("accept-agreement", {"help": "review & accept the Data Use Agreement (recorded locally)"}),
+        # Pre-2026-07-25 name, kept working; no `help` so it stays out of --help.
+        ("accept-terms", {}),
+    ):
+        p_acc = sub.add_parser(_name, **_kw)
+        p_acc.add_argument("-y", "--yes", action="store_true", help="accept without the interactive prompt")
+        p_acc.add_argument("--email", help="contact address to record (else ISAAC_AGREEMENT_EMAIL, else prompted)")
+        p_acc.add_argument("--status", action="store_true", help="show current acceptance record and exit")
+        p_acc.add_argument("--withdraw", action="store_true", help="delete the local acceptance record")
+
+    args = parser.parse_args(argv)
+
+    if args.cmd in ("accept-agreement", "accept-terms"):
+        from .agreement import accept_agreement, status, withdraw, AgreementNotAccepted
+        if args.status:
+            s = status()
+            print(json.dumps(s, indent=2) if s else "Data Use Agreement not yet accepted on this machine.")
+            return 0
+        if args.withdraw:
+            print("Removed local acceptance record." if withdraw() else "No acceptance record to remove.")
+            return 0
+        try:
+            accept_agreement(assume_yes=args.yes, email=args.email)
+            return 0
+        except AgreementNotAccepted as e:
+            print(str(e), file=sys.stderr)
+            return 2
+
+    if args.cmd == "ls":
+        df = files(args.category, args.start, args.end, _fmt(args))
+        cols = ["category", "year", "month", "format", "size_bytes", "num_rows", "url"]
+        with_pd_print(df[cols])
+        print(f"\n{len(df)} files, {df['size_bytes'].sum()/1e9:.2f} GB", file=sys.stderr)
+        return 0
+
+    if args.cmd == "info":
+        df = files(None, None, None, None)
+        import pandas as pd  # noqa
+        g = df.groupby(["category", "format"]).agg(files=("url", "size"),
+                                                   gb=("size_bytes", lambda s: round(s.sum() / 1e9, 1)))
+        with_pd_print(g.reset_index())
+        print(f"\nTotal: {len(df)} files, {df['size_bytes'].sum()/1e9:.1f} GB", file=sys.stderr)
+        return 0
+
+    if args.cmd == "download":
+        paths = download(args.category, args.start, args.end, _fmt(args), dest=args.dest)
+        for p in paths:
+            print(p)
+        print(f"\nDownloaded {len(paths)} file(s).", file=sys.stderr)
+        return 0
+
+    return 1
+
+
+def with_pd_print(df) -> None:
+    import pandas as pd
+    with pd.option_context("display.max_rows", 200, "display.width", 200):
+        print(df.to_string(index=False))
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())