TwoPy 0.1.0__py3-none-any.whl

twopy/database/cache.py

@@ -0,0 +1,166 @@
+"""Local cache support for Clark lab SQLite database files.
+
+Inputs: mounted SQLite DB paths and a local cache folder.
+Outputs: local DB copies plus JSON manifests that record source metadata.
+
+Copying exists for performance. Database queries over network-mounted volumes
+can be slow, but transferring the database file locally is usually fast.
+"""
+
+import hashlib
+import json
+import shutil
+from pathlib import Path
+from typing import cast
+
+from twopy.database.types import DatabaseAccess
+
+__all__ = [
+    "DEFAULT_DATABASE_CACHE_DIR",
+    "ensure_local_database_copy",
+    "validate_database_access",
+]
+
+DEFAULT_DATABASE_CACHE_DIR = Path.home() / ".cache" / "twopy" / "database"
+
+
+def validate_database_access(database_access: DatabaseAccess) -> None:
+    """Validate the database access mode.
+
+    Args:
+        database_access: Requested DB access mode.
+
+    Returns:
+        None when the mode is supported.
+
+    Raises:
+        ValueError: If the mode is not ``direct`` or ``copy``.
+    """
+    if database_access not in {"direct", "copy"}:
+        msg = f"database_access must be 'direct' or 'copy'; got {database_access!r}"
+        raise ValueError(msg)
+
+
+def ensure_local_database_copy(source_path: Path, cache_dir: Path) -> Path:
+    """Return a local cached copy of a SQLite database.
+
+    Args:
+        source_path: Source SQLite database path.
+        cache_dir: Directory where local copies and manifests are stored.
+
+    Returns:
+        Local cached database path.
+
+    The manifest stores source path, size, mtime, and SHA-256. If size and mtime
+    match, twopy reuses the local copy. If metadata changed, twopy hashes source
+    and cached files and copies only when content changed. This keeps repeated
+    queries fast without repeatedly transferring unchanged DB files.
+    """
+    cache_dir.mkdir(parents=True, exist_ok=True)
+    cache_path = cache_dir / source_path.name
+    manifest_path = cache_path.with_suffix(cache_path.suffix + ".manifest.json")
+    source_stat = source_path.stat()
+    source_metadata: dict[str, object] = {
+        "source_path": str(source_path),
+        "source_size": source_stat.st_size,
+        "source_mtime_ns": source_stat.st_mtime_ns,
+    }
+
+    manifest = _read_copy_manifest(manifest_path)
+    if cache_path.is_file() and _manifest_metadata_matches(manifest, source_metadata):
+        return cache_path
+
+    source_hash = _file_sha256(source_path)
+    cached_hash = _file_sha256(cache_path) if cache_path.is_file() else None
+    if source_hash != cached_hash:
+        temp_path = cache_path.with_suffix(cache_path.suffix + ".tmp")
+        shutil.copy2(source_path, temp_path)
+        temp_path.replace(cache_path)
+
+    _write_copy_manifest(
+        manifest_path,
+        {
+            **source_metadata,
+            "source_sha256": source_hash,
+        },
+    )
+    return cache_path
+
+
+def _read_copy_manifest(manifest_path: Path) -> dict[str, object] | None:
+    """Read a DB copy manifest.
+
+    Args:
+        manifest_path: Manifest JSON path.
+
+    Returns:
+        Parsed manifest dictionary or ``None`` when absent or invalid.
+
+    Invalid manifests are treated as cache misses because the source DB remains
+    the source of truth.
+    """
+    if not manifest_path.is_file():
+        return None
+
+    try:
+        loaded = json.loads(manifest_path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        return None
+
+    if not isinstance(loaded, dict):
+        return None
+
+    return cast(dict[str, object], loaded)
+
+
+def _manifest_metadata_matches(
+    manifest: dict[str, object] | None,
+    source_metadata: dict[str, object],
+) -> bool:
+    """Check whether source file metadata matches a manifest.
+
+    Args:
+        manifest: Parsed manifest or ``None``.
+        source_metadata: Current source path, size, and mtime metadata.
+
+    Returns:
+        ``True`` when the manifest points at the same unchanged source file.
+    """
+    if manifest is None:
+        return False
+
+    return all(manifest.get(key) == value for key, value in source_metadata.items())
+
+
+def _write_copy_manifest(manifest_path: Path, manifest: dict[str, object]) -> None:
+    """Write a DB copy manifest as stable JSON.
+
+    Args:
+        manifest_path: Manifest JSON path.
+        manifest: Manifest fields to write.
+
+    Returns:
+        None.
+    """
+    manifest_path.write_text(
+        json.dumps(manifest, indent=2, sort_keys=True) + "\n",
+        encoding="utf-8",
+    )
+
+
+def _file_sha256(path: Path) -> str:
+    """Compute SHA-256 for one file.
+
+    Args:
+        path: File path to hash.
+
+    Returns:
+        Hex SHA-256 digest.
+
+    The function streams chunks so database files do not need to fit in memory.
+    """
+    digest = hashlib.sha256()
+    with path.open("rb") as file:
+        while chunk := file.read(1024 * 1024):
+            digest.update(chunk)
+    return digest.hexdigest()

twopy/database/catalog.py

@@ -0,0 +1,90 @@
+"""Find and prepare Clark lab SQLite database files.
+
+Inputs: a lab database folder and an access mode.
+Outputs: a catalog of SQLite file paths, optionally copied to a local cache.
+"""
+
+from pathlib import Path
+
+from twopy.database.cache import (
+    DEFAULT_DATABASE_CACHE_DIR,
+    ensure_local_database_copy,
+    validate_database_access,
+)
+from twopy.database.types import (
+    DEFAULT_DATABASE_FILES,
+    DatabaseAccess,
+    DatabaseCatalog,
+)
+
+__all__ = ["read_database_catalog"]
+
+
+def read_database_catalog(
+    database_dir: Path,
+    *,
+    database_access: DatabaseAccess = "direct",
+    cache_dir: Path | None = None,
+) -> DatabaseCatalog:
+    """Find known SQLite database files in a database folder.
+
+    Args:
+        database_dir: Folder that should contain lab SQLite DB files.
+        database_access: ``direct`` reads DB files in place. ``copy`` uses local
+            cached copies because network DB queries can be slow while file
+            transfer is fast.
+        cache_dir: Optional local cache directory for copied DB files.
+
+    Returns:
+        A catalog listing known DB files that exist.
+
+    Raises:
+        FileNotFoundError: If the database folder is missing or no known DB file
+            exists.
+
+    The function checks only the known Clark lab DB filenames so macOS resource
+    fork files like ``._experimentLog.db`` are ignored.
+    """
+    root = database_dir.expanduser()
+    validate_database_access(database_access)
+
+    catalog = _read_direct_database_catalog(root)
+    if database_access == "direct":
+        return catalog
+
+    local_cache_dir = (cache_dir or DEFAULT_DATABASE_CACHE_DIR).expanduser()
+    return DatabaseCatalog(
+        database_dir=root,
+        sqlite_files=tuple(
+            ensure_local_database_copy(database_path, local_cache_dir)
+            for database_path in catalog.sqlite_files
+        ),
+    )
+
+
+def _read_direct_database_catalog(database_dir: Path) -> DatabaseCatalog:
+    """Find known SQLite files without copying them.
+
+    Args:
+        database_dir: Database folder to inspect.
+
+    Returns:
+        A catalog with direct database file paths.
+
+    Raises:
+        FileNotFoundError: If the folder or known DB files are absent.
+    """
+    root = database_dir.expanduser()
+    if not root.is_dir():
+        msg = f"Missing database directory: {root}"
+        raise FileNotFoundError(msg)
+
+    sqlite_files = tuple(
+        path for name in DEFAULT_DATABASE_FILES if (path := root / name).is_file()
+    )
+    if not sqlite_files:
+        expected = ", ".join(DEFAULT_DATABASE_FILES)
+        msg = f"No known SQLite database files found in {root}; expected {expected}"
+        raise FileNotFoundError(msg)
+
+    return DatabaseCatalog(database_dir=root, sqlite_files=sqlite_files)

twopy/database/connection.py

@@ -0,0 +1,27 @@
+"""Open SQLite databases with twopy's safety defaults.
+
+Inputs: SQLite database file paths.
+Outputs: read-only SQLite connections that return row dictionaries.
+"""
+
+import sqlite3
+from pathlib import Path
+
+__all__ = ["connect_read_only"]
+
+
+def connect_read_only(database_path: Path) -> sqlite3.Connection:
+    """Open one SQLite database in read-only mode.
+
+    Args:
+        database_path: SQLite file to open.
+
+    Returns:
+        SQLite connection configured for row dictionaries.
+
+    Read-only URI mode prevents accidental writes to the lab database.
+    """
+    uri = database_path.expanduser().resolve().as_uri() + "?mode=ro"
+    connection = sqlite3.connect(uri, uri=True, timeout=5)
+    connection.row_factory = sqlite3.Row
+    return connection

twopy/database/generic.py

@@ -0,0 +1,254 @@
+"""Schema-aware generic SQLite table search.
+
+Inputs: lab database folder, optional table names, and explicit filters.
+Outputs: ``DatabaseRecord`` rows with source DB and table provenance.
+
+This module supports exploratory queries before twopy has a modeled schema for a
+table.
+"""
+
+import sqlite3
+from collections.abc import Mapping, Sequence
+from contextlib import closing
+from pathlib import Path
+
+from twopy.database.catalog import read_database_catalog
+from twopy.database.connection import connect_read_only
+from twopy.database.sql import quote_identifier, validate_filter_columns
+from twopy.database.types import (
+    DatabaseAccess,
+    DatabaseColumn,
+    DatabaseRecord,
+    DatabaseTable,
+    DatabaseValue,
+)
+
+__all__ = ["find_experiments"]
+
+
+def find_experiments(
+    database_dir: Path,
+    *,
+    table_names: Sequence[str] | None = None,
+    exact_filters: Mapping[str, DatabaseValue] | None = None,
+    contains_filters: Mapping[str, str] | None = None,
+    search_text: str | None = None,
+    limit: int = 100,
+    database_access: DatabaseAccess = "direct",
+    cache_dir: Path | None = None,
+) -> tuple[DatabaseRecord, ...]:
+    """Find experiment rows across the known lab SQLite databases.
+
+    Args:
+        database_dir: Folder containing ``experimentLog.db`` and/or
+            ``experimentInitLog.db``.
+        table_names: Optional table names to search. ``None`` searches all user
+            tables in each DB.
+        exact_filters: Optional column-to-value equality filters.
+        contains_filters: Optional column-to-text substring filters.
+        search_text: Optional substring searched across all columns in each
+            selected table.
+        limit: Maximum number of records to return across all DB files.
+        database_access: ``direct`` queries DB files in place. ``copy`` queries
+            local cached copies because network DB queries can be slow while
+            file transfer is fast.
+        cache_dir: Optional cache directory used when ``database_access`` is
+            ``copy``.
+
+    Returns:
+        Matching records with source DB path, table name, and row values.
+
+    Raises:
+        FileNotFoundError: If no database files can be found.
+        ValueError: If filters reference columns absent from a selected table or
+            if ``limit`` is less than one.
+
+    Filters are combined with ``AND``. This conservative behavior keeps search
+    results auditable: every returned row satisfies every requested constraint.
+    """
+    if limit < 1:
+        msg = "Database query limit must be at least 1"
+        raise ValueError(msg)
+
+    catalog = read_database_catalog(
+        database_dir,
+        database_access=database_access,
+        cache_dir=cache_dir,
+    )
+    remaining = limit
+    records: list[DatabaseRecord] = []
+
+    for database_path in catalog.sqlite_files:
+        with closing(connect_read_only(database_path)) as connection:
+            tables = _matching_tables(connection, database_path, table_names)
+            for table in tables:
+                table_records = _query_table(
+                    connection=connection,
+                    table=table,
+                    exact_filters=exact_filters or {},
+                    contains_filters=contains_filters or {},
+                    search_text=search_text,
+                    limit=remaining,
+                )
+                records.extend(table_records)
+                remaining = limit - len(records)
+                if remaining == 0:
+                    return tuple(records)
+
+    return tuple(records)
+
+
+def _matching_tables(
+    connection: sqlite3.Connection,
+    database_path: Path,
+    table_names: Sequence[str] | None,
+) -> tuple[DatabaseTable, ...]:
+    """Return selected user tables from one SQLite database.
+
+    Args:
+        connection: Open SQLite connection.
+        database_path: Source DB path for returned metadata.
+        table_names: Optional table names requested by the caller.
+
+    Returns:
+        Table metadata for the selected tables.
+
+    Raises:
+        ValueError: If a requested table is absent.
+    """
+    available = _read_tables(connection, database_path)
+    if table_names is None:
+        return available
+
+    by_name = {table.name: table for table in available}
+    missing = tuple(name for name in table_names if name not in by_name)
+    if missing:
+        msg = f"Missing table(s) in {database_path}: {', '.join(missing)}"
+        raise ValueError(msg)
+
+    return tuple(by_name[name] for name in table_names)
+
+
+def _read_tables(
+    connection: sqlite3.Connection,
+    database_path: Path,
+) -> tuple[DatabaseTable, ...]:
+    """Read table and column metadata from one SQLite database.
+
+    Args:
+        connection: Open SQLite connection.
+        database_path: Source DB path for returned metadata.
+
+    Returns:
+        User tables sorted by name.
+
+    Internal SQLite tables are ignored because they are storage metadata, not
+    experiment data.
+    """
+    rows = connection.execute(
+        """
+        SELECT name
+        FROM sqlite_master
+        WHERE type = 'table'
+          AND name NOT LIKE 'sqlite_%'
+        ORDER BY name
+        """,
+    ).fetchall()
+
+    return tuple(
+        DatabaseTable(
+            database_path=database_path,
+            name=str(row["name"]),
+            columns=_read_columns(connection, str(row["name"])),
+        )
+        for row in rows
+    )
+
+
+def _read_columns(
+    connection: sqlite3.Connection,
+    table_name: str,
+) -> tuple[DatabaseColumn, ...]:
+    """Read column metadata for one SQLite table.
+
+    Args:
+        connection: Open SQLite connection.
+        table_name: Table to inspect.
+
+    Returns:
+        Column metadata in SQLite table order.
+    """
+    rows = connection.execute(
+        f"PRAGMA table_info({quote_identifier(table_name)})",
+    ).fetchall()
+    return tuple(
+        DatabaseColumn(
+            name=str(row["name"]),
+            sqlite_type=str(row["type"]),
+            is_primary_key=bool(row["pk"]),
+        )
+        for row in rows
+    )
+
+
+def _query_table(
+    *,
+    connection: sqlite3.Connection,
+    table: DatabaseTable,
+    exact_filters: Mapping[str, DatabaseValue],
+    contains_filters: Mapping[str, str],
+    search_text: str | None,
+    limit: int,
+) -> tuple[DatabaseRecord, ...]:
+    """Query one table with validated filters.
+
+    Args:
+        connection: Open SQLite connection.
+        table: Table metadata.
+        exact_filters: Equality filters.
+        contains_filters: Substring filters for named columns.
+        search_text: Optional substring searched across all columns.
+        limit: Maximum rows returned from this table.
+
+    Returns:
+        Matching records from this table.
+
+    SQL identifiers are quoted after validation against discovered columns. SQL
+    values are always passed as parameters.
+    """
+    column_names = tuple(column.name for column in table.columns)
+    validate_filter_columns(table, tuple(exact_filters))
+    validate_filter_columns(table, tuple(contains_filters))
+
+    where_clauses: list[str] = []
+    parameters: list[DatabaseValue] = []
+
+    for column, value in exact_filters.items():
+        where_clauses.append(f"{quote_identifier(column)} = ?")
+        parameters.append(value)
+
+    for column, value in contains_filters.items():
+        where_clauses.append(f"CAST({quote_identifier(column)} AS TEXT) LIKE ?")
+        parameters.append(f"%{value}%")
+
+    if search_text:
+        search_clause = " OR ".join(
+            f"CAST({quote_identifier(column)} AS TEXT) LIKE ?"
+            for column in column_names
+        )
+        where_clauses.append(f"({search_clause})")
+        parameters.extend(f"%{search_text}%" for _ in column_names)
+
+    where_sql = f" WHERE {' AND '.join(where_clauses)}" if where_clauses else ""
+    sql = f"SELECT * FROM {quote_identifier(table.name)}{where_sql} LIMIT ?"
+    parameters.append(limit)
+
+    rows = connection.execute(sql, parameters).fetchall()
+    return tuple(
+        DatabaseRecord(
+            database_path=table.database_path,
+            table_name=table.name,
+            values={column: row[column] for column in tuple(row.keys())},
+        )
+        for row in rows
+    )