aind-dynamic-foraging-database 0.0.1__py3-none-any.whl

aind_dynamic_foraging_database/__init__.py

@@ -0,0 +1,31 @@
+"""
+aind-dynamic-foraging-database — query (and build) the AIND dynamic-foraging parquet database.
+
+Querying is lightweight (just ``duckdb`` + ``pandas``). Reach for the helpers first; drop to
+native DuckDB SQL when you need more::
+
+    from aind_dynamic_foraging_database import select_sessions, fetch_trials
+    sel = select_sessions("task LIKE '%Uncoupled%' AND finished_rate > 0.9")
+    trials = fetch_trials(sel, columns=["animal_response", "earned_reward"])
+
+The default read targets (``SESSION_DB`` / ``TRIAL_DB`` / ``EVENT_DB``) live on a public S3 bucket,
+so reading needs no AWS credentials.
+
+Building/extending the database lives in ``build_cache`` / ``util.parquet_builder`` and needs the
+optional ``[build]`` extra (NWB readers + ``aind-dynamic-foraging-data-utils``); it is intentionally
+**not** imported here, so importing this package to *query* stays lightweight.
+"""
+
+__version__ = "0.0.1"
+
+from aind_dynamic_foraging_database.query import (  # noqa: F401
+    EVENT_DB,
+    PROD_S3_PREFIX,
+    SESSION_DB,
+    TRIAL_DB,
+    fetch_events,
+    fetch_trials,
+    read_events,
+    read_trials,
+    select_sessions,
+)

aind_dynamic_foraging_database/build_cache.py

@@ -0,0 +1,274 @@
+"""
+Main entry point to build (or incrementally extend) the foraging parquet cache.
+
+Runs the full pipeline over ALL sessions in the Han session table, exercising
+all three NWB routes (see references/data-sources.md):
+
+  - CO asset  -> AIND reader (nwb_utils) on the docDB S3 URI
+  - bonsai S3 -> legacy reader (Han bonsai NWB)
+  - bpod S3   -> legacy reader (Han bpod NWB)
+
+Incremental by default: only sessions not already recorded in
+``build_metadata.json`` are processed, so re-running cheaply adds new sessions.
+Pass ``--full-rebuild`` to reprocess everything.
+
+To query the built cache (the read-back / "return loop"), use the companion
+``query_examples`` module or ``query_examples.ipynb`` — querying is intentionally
+kept out of this build script.
+
+Output target:
+  - Default is a local scratch directory (safe for dev iteration).
+  - Point ``--out-dir`` at the canonical S3 prefix to write the production
+    database:  ``--out-dir s3://aind-scratch-data/aind-dynamic-foraging-cache``
+
+Run:
+    # incremental local build (default scratch dir)
+    python -m aind_dynamic_foraging_database.build_cache
+
+    # production build/update on S3 (--n-workers 64 ~= 4x faster; see --help)
+    python -m aind_dynamic_foraging_database.build_cache \\
+        --out-dir s3://aind-scratch-data/aind-dynamic-foraging-cache --n-workers 64
+
+    # quick smoke test on a random 300-session subset (spans all three routes)
+    python -m aind_dynamic_foraging_database.build_cache --limit 300
+
+Or drive it programmatically (the module is import-safe — nothing runs on import):
+    from aind_dynamic_foraging_database import build_cache as b
+    b.main(b.Config(out_dir="/root/capsule/scratch/tmp/foraging_cache", limit=300))
+"""
+
+import argparse
+import logging
+import os
+from dataclasses import dataclass
+from typing import Optional
+
+from aind_dynamic_foraging_database.util import parquet_builder
+
+logger = logging.getLogger(__name__)
+
+# Default output: local scratch (never /tmp). Use --out-dir for S3 prod.
+DEFAULT_OUT_DIR = "/root/capsule/scratch/tmp/foraging_cache"
+PROD_S3_OUT_DIR = "s3://aind-scratch-data/aind-dynamic-foraging-cache"
+
+
+@dataclass
+class Config:
+    """Inputs and derived output paths for one build."""
+
+    out_dir: str = DEFAULT_OUT_DIR
+    limit: Optional[int] = None  # cap sessions for a quick test (random subset)
+    full_rebuild: bool = False  # ignore build metadata; reprocess everything
+    random_seed: int = 42
+    n_workers: Optional[int] = None  # worker processes; None -> CO_CPUS-1
+    coalesce: bool = True  # merge each subject's sessions into one parquet file
+    co_cache: Optional[str] = None  # dev: cache the docDB discovery (pickle) here
+
+    @property
+    def is_s3(self) -> bool:
+        """Whether the output target is an S3 prefix (vs a local dir)."""
+        return self.out_dir.startswith("s3://")
+
+    @property
+    def session_out(self) -> str:
+        """Path to the flat session-table parquet."""
+        return f"{self.out_dir}/session_table.parquet"
+
+    @property
+    def trial_out(self) -> str:
+        """Prefix of the Hive-partitioned trial table."""
+        return f"{self.out_dir}/trial_table"
+
+    @property
+    def event_out(self) -> str:
+        """Prefix of the Hive-partitioned event table."""
+        return f"{self.out_dir}/event_table"
+
+    @property
+    def meta_out(self) -> str:
+        """Path to the incremental-build metadata JSON."""
+        return f"{self.out_dir}/build_metadata.json"
+
+    @property
+    def log_csv(self) -> str:
+        """Path to the human-readable per-session triage log CSV."""
+        return f"{self.out_dir}/processing_log.csv"
+
+
+# ---------------------------------------------------------------------------
+# Pipeline steps
+# ---------------------------------------------------------------------------
+
+
+def build_sessions(cfg: Config):
+    """
+    Build the complete session table: Han ∪ docDB/CO universe, CO assets attached.
+
+    See parquet_builder.build_session_table / _merge_han_and_co for the match
+    rule. The ~137 s docDB discovery can be cached locally for dev iteration via
+    --co-cache (loaded if present, else fetched once and saved).
+    """
+    _banner("Building session table (Han ∪ docDB CO universe)")
+    return parquet_builder.build_session_table(
+        output_path=cfg.session_out,
+        include_co_assets=True,
+        co_discovery=_load_or_fetch_co_discovery(cfg),
+        verbose=True,
+    )
+
+
+def _load_or_fetch_co_discovery(cfg: Config):
+    """
+    Dev helper: if --co-cache is set, load the cached docDB discovery (or fetch once and
+    save it). Returns None when no cache is configured, so build_session_table fetches fresh.
+
+    Cached as a **pickle** (not parquet): the docDB result has list-valued columns such as
+    ``co_task`` that pyarrow can't serialize, and pickle round-trips the full DataFrame
+    unchanged so a cached run matches a fresh fetch.
+    """
+    if not cfg.co_cache:
+        return None
+    import pandas as pd
+
+    if os.path.exists(cfg.co_cache):
+        print(f"  using cached docDB discovery: {cfg.co_cache}")
+        return pd.read_pickle(cfg.co_cache)
+
+    from aind_dynamic_foraging_data_utils.code_ocean_utils import get_dynamic_foraging_assets
+
+    print(f"  fetching docDB discovery, caching -> {cfg.co_cache}")
+    co = get_dynamic_foraging_assets()
+    co.to_pickle(cfg.co_cache)
+    return co
+
+
+def select_sessions(cfg: Config, session_df):
+    """
+    Optionally subsample the complete session table for a quick test (--limit N,
+    seeded). The full table is still written to session_out by build_sessions;
+    only the trial/event build is limited.
+    """
+    if cfg.limit is None or len(session_df) <= cfg.limit:
+        return session_df
+    sampled = session_df.sample(n=cfg.limit, random_state=cfg.random_seed)
+    print(f"  --limit: randomly sampled {len(sampled)} of {len(session_df)} sessions")
+    return sampled.reset_index(drop=True)
+
+
+def build_trial_event_tables(cfg: Config, session_df, nwb_index: dict) -> dict:
+    """Build the Hive-partitioned trial + event tables for the selected sessions."""
+    _banner("Building trial and event tables")
+    return parquet_builder.build_trial_and_event_tables(
+        session_df=session_df,
+        trial_output_prefix=cfg.trial_out,
+        event_output_prefix=cfg.event_out,
+        nwb_file_index=nwb_index,
+        build_metadata_path=cfg.meta_out,
+        incremental=not cfg.full_rebuild,
+        n_workers=cfg.n_workers,
+        coalesce=cfg.coalesce,
+        log_csv_path=cfg.log_csv,
+        verbose=True,
+    )
+
+
+def print_summary(cfg: Config, summary: dict) -> None:
+    """Print the build-result breakdown."""
+    _banner("BUILD SUMMARY")
+    print(f"  Output dir                   : {cfg.out_dir}")
+    print(f"  Processed (ok)               : {summary['n_processed']}")
+    print(f"  Skipped (no NWB found)       : {summary['n_skipped']}")
+    print(f"  Failed                       : {summary['n_failed']}")
+    print("\n  Data source breakdown:")
+    print(f"    CO asset                   : {summary['n_co_asset']}")
+    print(f"    Bonsai S3                  : {summary['n_bonsai_s3']}")
+    print(f"    bpod S3                    : {summary['n_bpod_s3']}")
+    print("\n  NWB reader breakdown:")
+    print(f"    AIND reader (CO asset)     : {summary['n_aind_reader']}")
+    print(f"    AIND->legacy fallback      : {summary['n_aind_fallback_legacy']}")
+    print(f"    Legacy bonsai              : {summary['n_legacy_bonsai']}")
+    print(f"    Legacy bpod                : {summary['n_legacy_bpod']}")
+    if summary["failed_sessions"]:
+        print(f"\n  Failed sessions ({summary['n_failed']}), first 20:")
+        for fs in summary["failed_sessions"][:20]:
+            print(f"    [{fs.get('data_source', '?')}] {fs['session_id']}  --  {fs['error']}")
+        if summary["n_failed"] > 20:
+            print(f"    ... and {summary['n_failed'] - 20} more")
+    print("=" * 60)
+
+
+# ---------------------------------------------------------------------------
+# Orchestration
+# ---------------------------------------------------------------------------
+
+
+def main(cfg: Config) -> dict:
+    """Run the full build pipeline end to end. Returns the build summary."""
+    logging.basicConfig(
+        level=logging.WARNING,
+        format="%(asctime)s %(levelname)s %(name)s: %(message)s",
+    )
+    # s3fs/aiobotocore emit benign "Task ... attached to a different loop"
+    # tracebacks when worker S3 clients are torn down (the read/write already
+    # succeeded). Quiet the asyncio logger so they don't clutter the build log.
+    logging.getLogger("asyncio").setLevel(logging.CRITICAL)
+    if not cfg.is_s3:
+        os.makedirs(cfg.out_dir, exist_ok=True)
+
+    _banner("Indexing local Han NWB files")
+    nwb_index = parquet_builder.build_nwb_file_index()
+    print(f"  Total NWB files indexed: {len(nwb_index)}")
+
+    # Build the complete session table (Han ∪ CO universe, CO assets attached),
+    # then optionally subsample for a quick --limit test, then build the tables.
+    session_df = build_sessions(cfg)
+    session_df = select_sessions(cfg, session_df)
+
+    summary = build_trial_event_tables(cfg, session_df, nwb_index)
+    print_summary(cfg, summary)
+    return summary
+
+
+def parse_args(argv=None) -> Config:
+    """Parse CLI arguments into a Config."""
+    p = argparse.ArgumentParser(description="Build/extend the foraging parquet cache.")
+    p.add_argument("--out-dir", default=DEFAULT_OUT_DIR,
+                   help=f"output dir or S3 prefix (default: %(default)s; "
+                        f"prod: {PROD_S3_OUT_DIR})")
+    p.add_argument("--limit", type=int, default=None,
+                   help="cap to a random N-session subset for a quick test (default: all)")
+    p.add_argument("--full-rebuild", action="store_true",
+                   help="ignore build metadata and reprocess every session")
+    p.add_argument("--n-workers", type=int, default=None,
+                   help="worker processes (default: CO_CPUS-1). CO-asset reads are "
+                        "I/O-bound, so oversubscribing past CPU count overlaps S3 "
+                        "latency. Recommended ~64 on a 16-core box: ~4x faster than "
+                        "the default, and beyond ~64 there's no gain (the "
+                        "create_df_trials parse saturates the cores). RAM is not "
+                        "the limit (~21 GB at 128 workers).")
+    p.add_argument("--no-coalesce", action="store_true",
+                   help="keep one parquet file per session instead of merging each "
+                        "subject's sessions into a single sorted file (the default)")
+    p.add_argument("--co-cache", default=None,
+                   help="dev: path to cache the ~137s docDB discovery "
+                        "(loaded if present, else fetched once and saved)")
+    args = p.parse_args(argv)
+    return Config(
+        out_dir=args.out_dir,
+        limit=args.limit,
+        full_rebuild=args.full_rebuild,
+        n_workers=args.n_workers,
+        coalesce=not args.no_coalesce,
+        co_cache=args.co_cache,
+    )
+
+
+def _banner(title: str) -> None:
+    """Print a titled banner to delimit pipeline stages in the log."""
+    print("\n" + "=" * 60)
+    print(title)
+    print("=" * 60)
+
+
+if __name__ == "__main__":
+    main(parse_args())

aind_dynamic_foraging_database/query.py

@@ -0,0 +1,284 @@
+"""
+DuckDB query helpers for the foraging parquet cache.
+
+Two layers — reach for the simple helpers first, drop to native SQL when you need more:
+
+  Layer 1 (convenience — the common "return loop"):
+      select_sessions -> fetch_trials / fetch_events
+    Filter the (small) session table on any metric / metadata, then pull those sessions'
+    trials or events with the session metadata already joined on — in one call.
+
+  Layer 0 (escape hatch — covers ANY query):
+      read_trials / read_events
+    Return a fast, partition-scoped ``read_parquet(...)`` clause for a set of subjects.
+    Drop it into whatever SQL you write — aggregations, window functions, trial<->event
+    joins, custom GROUP BY. You keep the full power of SQL; the helper only does the part
+    that is easy to get wrong or slow (reading the right partition files, fast + correct).
+
+Why scoped reads are fast: a full ``trial_table/**/*.parquet`` glob with ``union_by_name``
+must read *every* subject file's footer to build the column union before it can prune
+(~25 s cold). Scoping the read to just the subjects you asked for reads only their footers
+(~1 s), while still unioning their columns correctly.
+
+Everything reads the public S3 cache (no AWS credentials needed). To query a local build,
+pass ``base=`` (or reassign ``SESSION_DB`` / ``TRIAL_DB`` / ``EVENT_DB``).
+"""
+
+import duckdb
+
+PROD_S3_PREFIX = "s3://aind-scratch-data/aind-dynamic-foraging-cache"
+SESSION_DB = f"{PROD_S3_PREFIX}/session_table.parquet"  # flat session table
+TRIAL_DB = f"{PROD_S3_PREFIX}/trial_table"  # Hive-partitioned by subject_id
+EVENT_DB = f"{PROD_S3_PREFIX}/event_table"  # Hive-partitioned by subject_id
+
+# SELECT * over the trial table is ~21 GB — always project. These small defaults cover
+# the usual choice/reward analysis; pass columns=[...] for others, or columns="*" for all.
+DEFAULT_TRIAL_COLUMNS = [
+    "trial", "animal_response", "earned_reward",
+    "reward_probabilityL", "reward_probabilityR",
+]
+DEFAULT_EVENT_COLUMNS = ["trial", "timestamps", "event", "data"]
+
+# Leading identity columns we always emit and never duplicate from the trial/event side.
+_KEYS = ("subject_id", "session_date", "session_id")
+
+
+# ---------------------------------------------------------------------------
+# Internals
+# ---------------------------------------------------------------------------
+
+
+def _conn(con):
+    """Return the DuckDB connection to use (the given one, or the default module conn)."""
+    return con if con is not None else duckdb
+
+
+def _quote_in(values):
+    """Render an iterable as a SQL IN-list of quoted, escaped string literals."""
+    return ", ".join("'" + str(v).replace("'", "''") + "'" for v in values)
+
+
+def _partition_subjects(base, con=None):
+    """Subject ids that actually have a partition file under ``base``.
+
+    One cheap S3 LIST via ``glob()`` (not a footer scan) — used to drop requested
+    subjects with no files, since a scoped ``read_parquet`` list errors on a path that
+    matches nothing.
+    """
+    rows = _conn(con).sql(f"SELECT file FROM glob('{base}/subject_id=*/*.parquet')").df()
+    found = rows["file"].str.extract(r"subject_id=([^/]+)/")[0].dropna()
+    return set(found)
+
+
+def _full_glob(base):
+    """The correct-but-slow read over every subject (reads all footers for the union)."""
+    return f"read_parquet('{base}/**/*.parquet', hive_partitioning=true, union_by_name=true)"
+
+
+def _scoped_read(base, subjects, con):
+    """Build a ``read_parquet(...)`` clause scoped to ``subjects`` (or the full glob)."""
+    if subjects is None:
+        return _full_glob(base)
+    want = {str(s) for s in subjects} & _partition_subjects(base, con)
+    files = [f"'{base}/subject_id={s}/*.parquet'" for s in sorted(want)]
+    if not files:
+        # No requested subject has data: yield zero rows but the correct full schema.
+        return f"(SELECT * FROM {_full_glob(base)} WHERE false)"
+    return f"read_parquet([{', '.join(files)}], hive_partitioning=true, union_by_name=true)"
+
+
+# ---------------------------------------------------------------------------
+# Layer 0 — escape hatch: a fast, partition-scoped read_parquet(...) source
+# ---------------------------------------------------------------------------
+
+
+def read_trials(subjects=None, base=None, con=None):
+    """Return a ``read_parquet(...)`` clause for the trial table, scoped to ``subjects``.
+
+    Drop the returned string into any SQL you write::
+
+        src = read_trials(['754372', '758435'])
+        duckdb.sql(f"SELECT subject_id, AVG(earned_reward::DOUBLE) FROM {src} GROUP BY subject_id")
+
+    Scoping to the subjects you need reads only their partition files (~1 s) instead of
+    every subject's footer. ``subjects=None`` falls back to the full (slow) glob over all
+    subjects. Note a scoped read exposes only the columns present in *those* subjects'
+    files; selecting a column none of them has will raise.
+
+    Parameters
+    ----------
+    subjects : iterable, optional
+        Subject ids to scope the read to. ``None`` reads the full table (slow glob).
+    base : str, optional
+        Trial-table location — the partitioned-table **directory** prefix (default: the
+        production S3 ``trial_table``). Pass a local dir / other S3 prefix for another build.
+    con : duckdb connection, optional
+        DuckDB connection to run the partition listing on (default: the module connection).
+        Pass your own for warm reuse, or custom settings (S3 region/creds, threads, memory).
+    """
+    return _scoped_read(base or TRIAL_DB, subjects, con)
+
+
+def read_events(subjects=None, base=None, con=None):
+    """Return a ``read_parquet(...)`` clause for the event table, scoped to ``subjects``.
+
+    The event-table counterpart of :func:`read_trials` — same ``subjects`` / ``base`` / ``con``
+    behaviour, except ``base`` defaults to the production S3 ``event_table`` directory prefix.
+    """
+    return _scoped_read(base or EVENT_DB, subjects, con)
+
+
+# ---------------------------------------------------------------------------
+# Layer 1 — convenience: filter sessions, then fetch their trials / events
+# ---------------------------------------------------------------------------
+
+
+def select_sessions(where=None, subjects=None, columns=None, base=None, con=None,
+                    order_by="subject_id, session_date"):
+    """Filter the (small) session table; return the selected sessions as a DataFrame.
+
+    The first step of both common workflows — filter on session metrics/metadata, or on
+    subject first, or both — then hand the result to :func:`fetch_trials` /
+    :func:`fetch_events`.
+
+    Parameters
+    ----------
+    where : str, optional
+        Raw SQL predicate on the session table, e.g.
+        ``"task LIKE '%Uncoupled%' AND foraging_eff > 0.8"``.
+    subjects : iterable, optional
+        Restrict to these subject ids (adds ``subject_id IN (...)``).
+    columns : list[str], optional
+        Extra session-metadata columns to carry along (and onto trials/events later).
+        ``_session_id, subject_id, session_date`` are always included as leading columns.
+    base : str, optional
+        Session table to read — the ``session_table.parquet`` **file** (default: the
+        production S3 cache). Pass a local file / other S3 path to query another build.
+    con : duckdb connection, optional
+        DuckDB connection to run on (default: the module connection). Pass your own for warm
+        reuse across calls, or custom settings (S3 region/creds, threads, memory).
+    order_by : str, optional
+        SQL ORDER BY clause (default: ``"subject_id, session_date"``); pass ``None`` for none.
+
+    Returns
+    -------
+    pandas.DataFrame
+        One row per selected session, with ``_session_id`` as the join key.
+    """
+    base = base or SESSION_DB
+    extra = [c for c in (columns or []) if c not in ("_session_id", *_KEYS)]
+    sel_cols = ", ".join(["_session_id", "subject_id", "session_date", *extra])
+    clauses = []
+    if subjects is not None:
+        clauses.append(f"subject_id IN ({_quote_in(subjects)})")
+    if where:
+        clauses.append(f"({where})")
+    where_sql = ("WHERE " + " AND ".join(clauses)) if clauses else ""
+    order_sql = f"ORDER BY {order_by}" if order_by else ""
+    return _conn(con).sql(
+        f"SELECT {sel_cols} FROM read_parquet('{base}') {where_sql} {order_sql}"
+    ).df()
+
+
+def fetch_trials(sessions, columns=None, base=None, con=None):
+    """Pull trial rows for a set of selected sessions, with session metadata joined on.
+
+    Reads only the selected subjects' partitions (fast) and inner-joins to ``sessions`` on
+    the session key, so exactly the selected sessions' trials are returned — each row
+    carrying its session metadata.
+
+    Parameters
+    ----------
+    sessions : pandas.DataFrame
+        Selected sessions (e.g. from :func:`select_sessions`). Must contain ``_session_id``
+        and ``subject_id``; every other column is carried onto each trial row.
+    columns : list[str] or "*", optional
+        Trial columns to project (default: a small choice/reward set). ``"*"`` returns all
+        103 columns (large). Columns absent for the selected subjects come back all-NULL.
+    base : str, optional
+        Trial-table **directory** prefix (default: the production S3 ``trial_table``). Pass a
+        local dir / other S3 prefix to query another build.
+    con : duckdb connection, optional
+        DuckDB connection to run on (default: the module connection). Pass your own for warm
+        reuse across calls, or custom settings (S3 region/creds, threads, memory).
+
+    Returns
+    -------
+    pandas.DataFrame
+        One row per trial, leading ``subject_id, session_date, session_id``, ordered by
+        ``subject_id, session_date, trial``.
+    """
+    return _fetch(sessions, base or TRIAL_DB, columns or DEFAULT_TRIAL_COLUMNS,
+                  con, order_tail="trial")
+
+
+def fetch_events(sessions, events=None, columns=None, base=None, con=None):
+    """Pull event rows for a set of selected sessions, with session metadata joined on.
+
+    Like :func:`fetch_trials`, for the event table.
+
+    Parameters
+    ----------
+    sessions : pandas.DataFrame
+        Selected sessions (needs ``_session_id`` and ``subject_id``).
+    events : iterable, optional
+        Restrict to these event types, e.g. ``['left_lick_time', 'right_lick_time']``.
+    columns : list[str] or "*", optional
+        Event columns to project (default: ``trial, timestamps, event, data``). Columns absent
+        for the selected subjects come back all-NULL.
+    base : str, optional
+        Event-table **directory** prefix (default: the production S3 ``event_table``). Pass a
+        local dir / other S3 prefix to query another build.
+    con : duckdb connection, optional
+        DuckDB connection to run on (default: the module connection). Pass your own for warm
+        reuse across calls, or custom settings (S3 region/creds, threads, memory).
+
+    Returns
+    -------
+    pandas.DataFrame
+        One row per event, leading ``subject_id, session_date, session_id``, ordered by
+        ``subject_id, session_date, timestamps``.
+    """
+    extra_where = f"t.event IN ({_quote_in(events)})" if events else None
+    return _fetch(sessions, base or EVENT_DB, columns or DEFAULT_EVENT_COLUMNS,
+                  con, order_tail="timestamps", extra_where=extra_where)
+
+
+def _fetch(sessions, base, columns, con, order_tail, extra_where=None):
+    """Shared core for fetch_trials / fetch_events: scoped read + join to selected sessions.
+
+    A scoped read exposes only the columns present in *those* subjects' files (some columns
+    are reader-specific, e.g. ``trial`` is absent from some legacy files). So we adapt to the
+    columns actually available: requested columns that are missing are emitted as all-NULL
+    (stable output shape, never an error), and the ORDER BY tail is dropped if absent.
+    """
+    import pandas as pd
+
+    if len(sessions) == 0:
+        return pd.DataFrame()
+    conn = _conn(con)
+    src = _scoped_read(base, sessions["subject_id"].unique().tolist(), con)
+    avail = set(conn.sql(f"DESCRIBE SELECT * FROM {src}").df()["column_name"])
+    conn.register("_sel_sessions", sessions)
+
+    meta = [f"s.{c}" for c in sessions.columns if c not in ("_session_id", *_KEYS)]
+    if columns in ("*", ["*"]):
+        proj = [f"t.* EXCLUDE ({', '.join(k for k in _KEYS if k in avail)})"]
+    else:
+        proj = [f"t.{c}" if c in avail else f"CAST(NULL AS DOUBLE) AS {c}"
+                for c in columns if c not in _KEYS]
+    select = ", ".join(["s.subject_id", "s.session_date", "t.session_id", *meta, *proj])
+    where_sql = f"WHERE {extra_where}" if extra_where else ""
+    order = ["s.subject_id", "s.session_date"]
+    if order_tail in avail:
+        order.append(f"t.{order_tail}")
+    try:
+        return conn.sql(f"""
+            SELECT {select}
+            FROM {src} t
+            JOIN _sel_sessions s ON t.session_id = s._session_id
+            {where_sql}
+            ORDER BY {', '.join(order)}
+        """).df()
+    finally:
+        conn.unregister("_sel_sessions")