w2t-bkin 0.0.6__py3-none-any.whl

w2t_bkin/config.py

@@ -0,0 +1,625 @@
+"""Configuration management for W2T-BKIN pipeline.
+
+This module provides Pydantic models for validating configuration files (config.toml)
+and functions for loading, validating, and hashing configurations.
+
+The configuration system enforces strict schema validation to catch errors early,
+supports deterministic hashing for reproducibility, and provides clear error messages.
+
+Typical usage example:
+    >>> from w2t_bkin.config import load_config
+    >>>
+    >>> config = load_config("config.toml")
+    >>> print(config.project.name)
+    >>> print(config.timebase.source)
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Dict, List, Literal, Optional, Union
+
+try:
+    import tomllib
+except ImportError:
+    import tomli as tomllib  # Python < 3.11 fallback
+
+from pydantic import BaseModel, Field, ValidationError, field_validator
+
+from .utils import compute_hash, read_toml
+
+# =============================================================================
+# Constants
+# =============================================================================
+
+VALID_TIMEBASE_SOURCES = frozenset({"nominal_rate", "ttl", "neuropixels"})
+VALID_TIMEBASE_MAPPINGS = frozenset({"nearest", "linear"})
+VALID_LOGGING_LEVELS = frozenset({"DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"})
+
+
+# =============================================================================
+# Configuration Models - Core
+# =============================================================================
+
+
+class ProjectConfig(BaseModel, extra="forbid"):
+    """Project identification.
+
+    Attributes:
+        name: Project name identifier.
+    """
+
+    name: str = Field(..., description="Project name")
+
+
+class PathsConfig(BaseModel, extra="forbid"):
+    """File system paths configuration.
+
+    Attributes:
+        raw_root: Path to raw data directory.
+        intermediate_root: Path for intermediate processing outputs.
+        output_root: Path for final outputs.
+        metadata_file: Filename for session metadata (default: session.toml).
+        models_root: Directory containing pose estimation models (default: models).
+    """
+
+    raw_root: Path = Field(..., description="Raw data root directory")
+    intermediate_root: Path = Field(..., description="Intermediate processing outputs")
+    output_root: Path = Field(..., description="Output data root directory")
+    metadata_file: Path = Field(default="session.toml", description="Session metadata filename")
+    models_root: Path = Field(default="models", description="Pose estimation models directory")
+
+
+class TimebaseConfig(BaseModel, extra="forbid"):
+    """Reference timebase for aligning derived data.
+
+    Defines the reference clock for synchronizing pose and behavior data.
+    ImageSeries remain rate-based; this timebase applies to derived modalities.
+
+    Attributes:
+        source: Timebase source (nominal_rate, ttl, or neuropixels).
+        mapping: Strategy for mapping timestamps (nearest or linear).
+        jitter_budget_s: Maximum allowed temporal jitter in seconds.
+        offset_s: Global time offset before mapping (default: 0.0).
+        ttl_id: TTL channel ID (required when source='ttl').
+        neuropixels_stream: Neuropixels stream name (required when source='neuropixels').
+    """
+
+    source: Literal["nominal_rate", "ttl", "neuropixels"] = Field(..., description="Timebase source")
+    mapping: Literal["nearest", "linear"] = Field(..., description="Mapping strategy")
+    jitter_budget_s: float = Field(..., ge=0.0, description="Max allowed jitter in seconds")
+    offset_s: float = Field(default=0.0, description="Global offset before mapping")
+    ttl_id: Optional[str] = Field(None, description="TTL ID (required when source='ttl')")
+    neuropixels_stream: Optional[str] = Field(None, description="Neuropixels stream (required when source='neuropixels')")
+
+
+class AcquisitionConfig(BaseModel, extra="forbid"):
+    """Data acquisition policies.
+
+    Attributes:
+        concat_strategy: Video concatenation method (ffconcat or streamlist).
+    """
+
+    concat_strategy: Literal["ffconcat", "streamlist"] = Field(default="ffconcat", description="Video concatenation strategy")
+
+
+class VerificationConfig(BaseModel, extra="forbid"):
+    """Hardware synchronization verification.
+
+    Attributes:
+        mismatch_tolerance_frames: Max allowed frame/TTL count mismatch before abort.
+        warn_on_mismatch: If True, warn instead of abort when within tolerance.
+    """
+
+    mismatch_tolerance_frames: int = Field(default=0, ge=0, description="Abort if frame_count - ttl_pulse_count > tolerance")
+    warn_on_mismatch: bool = Field(default=False, description="Warn instead of abort if within tolerance")
+
+
+# =============================================================================
+# Configuration Models - Bpod
+# =============================================================================
+
+
+class BpodSyncTrialType(BaseModel, extra="forbid"):
+    """Bpod trial type synchronization mapping.
+
+    Maps a Bpod trial type to its synchronization signal and TTL channel,
+    enabling conversion from Bpod relative timestamps to absolute time.
+
+    Attributes:
+        trial_type: Trial type identifier matching Bpod classification.
+        sync_signal: Bpod state/event name for alignment (e.g., 'W2T_Audio').
+        sync_ttl: TTL channel whose pulses correspond to sync_signal.
+    """
+
+    trial_type: int = Field(..., ge=0, description="Trial type identifier")
+    sync_signal: str = Field(..., description="Bpod state/event for alignment")
+    sync_ttl: str = Field(..., description="TTL channel for sync pulses")
+
+
+class BpodSyncConfig(BaseModel, extra="forbid"):
+    """Bpod-to-TTL synchronization configuration.
+
+    Attributes:
+        trial_types: List of trial type sync configurations.
+    """
+
+    trial_types: List[BpodSyncTrialType] = Field(default_factory=list, description="Trial type sync configs")
+
+
+class BpodConfig(BaseModel, extra="forbid"):
+    """Bpod behavioral control system configuration.
+
+    Attributes:
+        parse: Whether to parse Bpod .mat files.
+        sync: Trial synchronization configuration.
+    """
+
+    parse: bool = Field(default=True, description="Parse Bpod .mat files if present")
+    sync: BpodSyncConfig = Field(default_factory=BpodSyncConfig, description="Trial sync configuration")
+
+
+# =============================================================================
+# Configuration Models - Video
+# =============================================================================
+
+
+class TranscodeConfig(BaseModel, extra="forbid"):
+    """Video transcoding settings.
+
+    Attributes:
+        enabled: Enable video transcoding.
+        codec: FFmpeg codec (e.g., 'h264', 'libx264').
+        crf: Constant rate factor quality (0-51, lower is better).
+        preset: FFmpeg encoding preset (e.g., 'fast', 'medium').
+        keyint: GOP (group of pictures) length.
+    """
+
+    enabled: bool = Field(default=True, description="Enable transcoding")
+    codec: str = Field(default="h264", description="FFmpeg codec name")
+    crf: int = Field(default=20, ge=0, le=51, description="Quality factor (0-51)")
+    preset: str = Field(default="fast", description="FFmpeg preset")
+    keyint: int = Field(default=15, ge=1, description="GOP length")
+
+
+class VideoConfig(BaseModel, extra="forbid"):
+    """Video processing configuration.
+
+    Attributes:
+        transcode: Transcoding settings.
+    """
+
+    transcode: TranscodeConfig = Field(default_factory=TranscodeConfig, description="Transcoding config")
+
+
+# =============================================================================
+# Configuration Models - Output & Logging
+# =============================================================================
+
+
+class NWBConfig(BaseModel, extra="forbid"):
+    """NWB (Neurodata Without Borders) export settings.
+
+    Attributes:
+        link_external_video: Use external links for videos instead of embedding.
+        lab: Laboratory name.
+        institution: Institution name.
+        file_name_template: Template for NWB filename.
+        session_description_template: Template for session description.
+    """
+
+    link_external_video: bool = Field(default=True, description="Link videos externally")
+    lab: str = Field(default="Lab Name", description="Lab name")
+    institution: str = Field(default="Institution Name", description="Institution name")
+    file_name_template: str = Field(default="{session.id}.nwb", description="NWB filename template")
+    session_description_template: str = Field(default="Session {session.id} on {session.date}", description="Session description template")
+
+
+class QCConfig(BaseModel, extra="forbid"):
+    """Quality control report configuration.
+
+    Attributes:
+        generate_report: Enable QC report generation.
+        out_template: Output path template for reports.
+        include_verification: Include frame/TTL verification in reports.
+    """
+
+    generate_report: bool = Field(default=True, description="Generate QC report")
+    out_template: str = Field(default="qc/{session.id}", description="Output path template")
+    include_verification: bool = Field(default=True, description="Include verification in report")
+
+
+class LoggingConfig(BaseModel, extra="forbid"):
+    """Logging configuration.
+
+    Attributes:
+        level: Logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL).
+        structured: Use structured (JSON) logging format.
+    """
+
+    level: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] = Field(default="INFO", description="Logging level")
+    structured: bool = Field(default=False, description="Use structured logging")
+
+
+# =============================================================================
+# Configuration Models - Inference
+# =============================================================================
+
+
+class DLCConfig(BaseModel, extra="forbid"):
+    """DeepLabCut pose estimation configuration.
+
+    Attributes:
+        run_inference: Enable DLC inference.
+        model: Path to DLC model file.
+        gputouse: GPU device index (-1 for CPU, None for auto-select).
+    """
+
+    run_inference: bool = Field(default=False, description="Run DLC inference")
+    model: str = Field(default="model.pb", description="DLC model path")
+    gputouse: Optional[int] = Field(None, description="GPU index (-1=CPU, None=auto)")
+
+
+class SLEAPConfig(BaseModel, extra="forbid"):
+    """SLEAP pose estimation configuration.
+
+    Attributes:
+        run_inference: Enable SLEAP inference.
+        model: Path to SLEAP model file.
+    """
+
+    run_inference: bool = Field(default=False, description="Run SLEAP inference")
+    model: str = Field(default="sleap.h5", description="SLEAP model path")
+
+
+class LabelsConfig(BaseModel, extra="forbid"):
+    """Pose labeling configuration.
+
+    Attributes:
+        dlc: DeepLabCut configuration.
+        sleap: SLEAP configuration.
+    """
+
+    dlc: DLCConfig = Field(default_factory=DLCConfig, description="DLC config")
+    sleap: SLEAPConfig = Field(default_factory=SLEAPConfig, description="SLEAP config")
+
+
+class FacemapConfig(BaseModel, extra="forbid"):
+    """Facemap facial motion tracking configuration.
+
+    Attributes:
+        run_inference: Enable Facemap inference.
+        ROIs: Regions of interest to process.
+    """
+
+    run_inference: bool = Field(default=False, description="Run Facemap inference")
+    ROIs: List[str] = Field(default_factory=lambda: ["face", "left_eye", "right_eye"], description="ROIs to process")
+
+
+# =============================================================================
+# Main Configuration Model
+# =============================================================================
+
+
+class Config(BaseModel, extra="forbid"):
+    """Main pipeline configuration.
+
+    Root configuration model loaded from config.toml. Uses strict validation
+    with extra="forbid" to prevent typos and configuration errors.
+
+    Attributes:
+        project: Project identification.
+        paths: File system paths.
+        timebase: Reference timebase for synchronization.
+        acquisition: Data acquisition policies.
+        verification: Hardware sync verification.
+        bpod: Bpod behavioral control settings.
+        video: Video processing configuration.
+        nwb: NWB export settings.
+        qc: Quality control configuration.
+        logging: Logging configuration.
+        labels: Pose labeling configuration.
+        facemap: Facemap tracking configuration.
+    """
+
+    project: ProjectConfig
+    paths: PathsConfig
+    # timebase: TimebaseConfig
+    # acquisition: AcquisitionConfig = Field(default_factory=AcquisitionConfig)
+    # verification: VerificationConfig = Field(default_factory=VerificationConfig)
+    bpod: BpodConfig = Field(default_factory=BpodConfig)
+    # video: VideoConfig = Field(default_factory=VideoConfig)
+    # nwb: NWBConfig = Field(default_factory=NWBConfig)
+    # qc: QCConfig = Field(default_factory=QCConfig)
+    logging: LoggingConfig = Field(default_factory=LoggingConfig)
+    # labels: LabelsConfig = Field(default_factory=LabelsConfig)
+    # facemap: FacemapConfig = Field(default_factory=FacemapConfig)
+
+    # @field_validator("timebase")
+    # @classmethod
+    # def validate_timebase_conditionals(cls, v: TimebaseConfig) -> TimebaseConfig:
+    #     """Validate conditional timebase requirements.
+
+    #     Args:
+    #         v: TimebaseConfig instance to validate.
+
+    #     Returns:
+    #         Validated TimebaseConfig.
+
+    #     Raises:
+    #         ValueError: If conditional requirements are not met.
+    #     """
+    #     if v.source == "ttl" and v.ttl_id is None:
+    #         raise ValueError("timebase.ttl_id is required when source='ttl'")
+    #     if v.source == "neuropixels" and v.neuropixels_stream is None:
+    #         raise ValueError("timebase.neuropixels_stream is required when source='neuropixels'")
+    #     return v
+
+
+# =============================================================================
+# Public API Functions
+# =============================================================================
+
+
+def load_config(path: Union[str, Path]) -> Config:
+    """Load and validate configuration from TOML file.
+
+    Performs comprehensive validation including:
+    - Schema validation with extra="forbid" to prevent typos
+    - Enum validation for source, mapping, and level fields
+    - Numeric constraints (e.g., jitter_budget_s >= 0)
+    - Conditional requirements (e.g., ttl_id when source='ttl')
+
+    Args:
+        path: Path to config.toml file.
+
+    Returns:
+        Validated Config instance.
+
+    Raises:
+        FileNotFoundError: If config file doesn't exist.
+        ValidationError: If config violates Pydantic schema.
+        ValueError: If enum or conditional validation fails.
+
+    Example:
+        >>> config = load_config("config.toml")
+        >>> print(config.project.name)
+        >>> print(config.timebase.source)
+    """
+    data = read_toml(path)
+
+    # Pre-validate enums for clearer error messages
+    _validate_config_enums(data)
+
+    # Pre-validate conditional requirements
+    _validate_config_conditionals(data)
+
+    return Config(**data)
+
+
+def compute_config_hash(config: Config) -> str:
+    """Compute deterministic SHA256 hash of configuration.
+
+    Converts config to canonical dict representation and computes hash.
+    Useful for tracking configuration changes and ensuring reproducibility.
+
+    Args:
+        config: Config instance to hash.
+
+    Returns:
+        SHA256 hex digest (64 characters).
+
+    Example:
+        >>> config = load_config("config.toml")
+        >>> hash_value = compute_config_hash(config)
+        >>> print(f"Config hash: {hash_value[:16]}...")
+    """
+    config_dict = config.model_dump()
+    return compute_hash(config_dict)
+
+
+# =============================================================================
+# Private Validation Helpers
+# =============================================================================
+
+
+def _validate_config_enums(data: Dict[str, Any]) -> None:
+    """Validate enum constraints before Pydantic validation.
+
+    Pre-validates enum fields to provide clearer error messages than
+    Pydantic's default validation.
+
+    Args:
+        data: Raw configuration dict from TOML.
+
+    Raises:
+        ValueError: If any enum value is invalid.
+    """
+    timebase = data.get("timebase", {})
+
+    # Validate timebase.source
+    source = timebase.get("source")
+    if source and source not in VALID_TIMEBASE_SOURCES:
+        raise ValueError(f"Invalid timebase.source: '{source}'. " f"Must be one of {sorted(VALID_TIMEBASE_SOURCES)}")
+
+    # Validate timebase.mapping
+    mapping = timebase.get("mapping")
+    if mapping and mapping not in VALID_TIMEBASE_MAPPINGS:
+        raise ValueError(f"Invalid timebase.mapping: '{mapping}'. " f"Must be one of {sorted(VALID_TIMEBASE_MAPPINGS)}")
+
+    # Validate jitter_budget_s >= 0
+    jitter_budget = timebase.get("jitter_budget_s")
+    if jitter_budget is not None and jitter_budget < 0:
+        raise ValueError(f"Invalid timebase.jitter_budget_s: {jitter_budget}. " f"Must be >= 0")
+
+    # Validate logging.level
+    logging_config = data.get("logging", {})
+    level = logging_config.get("level")
+    if level and level not in VALID_LOGGING_LEVELS:
+        raise ValueError(f"Invalid logging.level: '{level}'. " f"Must be one of {sorted(VALID_LOGGING_LEVELS)}")
+
+
+def _validate_config_conditionals(data: Dict[str, Any]) -> None:
+    """Validate conditional requirements before Pydantic validation.
+
+    Checks that required fields are present based on other field values.
+
+    Args:
+        data: Raw configuration dict from TOML.
+
+    Raises:
+        ValueError: If conditional requirements are not met.
+    """
+    timebase = data.get("timebase", {})
+    source = timebase.get("source")
+
+    if source == "ttl" and not timebase.get("ttl_id"):
+        raise ValueError("timebase.ttl_id is required when timebase.source='ttl'")
+
+    if source == "neuropixels" and not timebase.get("neuropixels_stream"):
+        raise ValueError("timebase.neuropixels_stream is required when " "timebase.source='neuropixels'")
+
+
+# =============================================================================
+# Session Loading Functions (backward compatibility)
+# =============================================================================
+
+
+def load_session(path: Union[str, Path]) -> Dict[str, Any]:
+    """Load session metadata from TOML file.
+
+    Args:
+        path: Path to session.toml or metadata.toml file.
+
+    Returns:
+        Parsed session metadata dictionary.
+
+    Raises:
+        FileNotFoundError: If file doesn't exist.
+
+    Example:
+        >>> session = load_session("data/raw/Session-000001/session.toml")
+        >>> print(session["identifier"])
+    """
+    session_path = Path(path)
+
+    if not session_path.exists():
+        raise FileNotFoundError(f"Session file not found: {session_path}")
+
+    return read_toml(session_path)
+
+
+def compute_session_hash(session: Dict[str, Any]) -> str:
+    """Compute deterministic SHA256 hash of session metadata.
+
+    Args:
+        session: Session metadata dictionary.
+
+    Returns:
+        SHA256 hex digest (64 characters).
+
+    Example:
+        >>> session = load_session("session.toml")
+        >>> hash_value = compute_session_hash(session)
+        >>> print(f"Session hash: {hash_value[:16]}...")
+    """
+    return compute_hash(session)
+
+
+# =============================================================================
+# CLI/Testing Entry Point
+# =============================================================================
+
+if __name__ == "__main__":
+    """Demonstrate configuration loading and validation."""
+
+    print("=" * 70)
+    print("Configuration Loading Examples")
+    print("=" * 70)
+    print()
+
+    # Example 1: Load valid configuration
+    print("Example 1: Load and validate config.toml")
+    print("-" * 70)
+
+    try:
+        config_path = Path("tests/fixtures/configs/valid_config.toml")
+        config = load_config(config_path)
+
+        print(f"✓ Loaded: {config_path}")
+        print(f"  Project: {config.project.name}")
+        print(f"  Timebase: {config.timebase.source} ({config.timebase.mapping})")
+        print(f"  Jitter budget: {config.timebase.jitter_budget_s}s")
+        print(f"  Logging: {config.logging.level}")
+
+        config_hash = compute_config_hash(config)
+        print(f"  Hash: {config_hash[:16]}...")
+
+    except FileNotFoundError as e:
+        print(f"✗ File not found: {e}")
+        print("  Hint: Run from project root")
+    except ValidationError as e:
+        print(f"✗ Validation failed:")
+        for error in e.errors():
+            print(f"  - {error['loc']}: {error['msg']}")
+    except ValueError as e:
+        print(f"✗ Configuration error: {e}")
+
+    print()
+
+    # Example 2: Demonstrate validation errors
+    print("Example 2: Validation error handling")
+    print("-" * 70)
+
+    # Invalid enum
+    print("\n2a. Invalid timebase.source:")
+    try:
+        test_data = {
+            "project": {"name": "test"},
+            "paths": {
+                "raw_root": "data/raw",
+                "intermediate_root": "data/interim",
+                "output_root": "data/processed",
+            },
+            "timebase": {
+                "source": "invalid",
+                "mapping": "nearest",
+                "jitter_budget_s": 0.01,
+            },
+        }
+        _validate_config_enums(test_data)
+    except ValueError as e:
+        print(f"  ✓ Caught: {e}")
+
+    # Missing conditional field
+    print("\n2b. Missing conditional field (ttl_id):")
+    try:
+        test_data = {
+            "timebase": {
+                "source": "ttl",
+                "mapping": "nearest",
+                "jitter_budget_s": 0.01,
+            }
+        }
+        _validate_config_conditionals(test_data)
+    except ValueError as e:
+        print(f"  ✓ Caught: {e}")
+
+    # Invalid numeric constraint
+    print("\n2c. Invalid numeric constraint:")
+    try:
+        test_data = {
+            "timebase": {
+                "source": "nominal_rate",
+                "mapping": "nearest",
+                "jitter_budget_s": -0.01,
+            }
+        }
+        _validate_config_enums(test_data)
+    except ValueError as e:
+        print(f"  ✓ Caught: {e}")
+
+    print()
+    print("=" * 70)
+    print("See module docstring for more information")
+    print("=" * 70)

w2t_bkin/dlc/__init__.py

@@ -0,0 +1,59 @@
+"""DLC (DeepLabCut) inference module.
+
+This module provides low-level primitives for running DeepLabCut model inference
+on video files. It follows the 3-tier architecture:
+
+- **Low-level**: Functions accept primitives only (Path, int, bool, List)
+- **No Config/Session**: Never imports config, Session, or Manifest
+- **Module-local models**: Owns DLCInferenceOptions, DLCInferenceResult, DLCModelInfo
+
+**Key Features**:
+- Batch processing: Single DLC call for multiple videos (optimal GPU utilization)
+- GPU auto-detection: Automatic GPU selection with manual override support
+- Partial failure handling: Gracefully handle individual video failures in batch
+- Idempotency: Content-addressed outputs, skip inference if unchanged
+
+**Architecture**:
+- ``dlc/core.py``: Low-level inference functions
+- ``dlc/models.py``: Module-local data models
+- ``dlc/__init__.py``: Public API surface
+
+Requirements:
+    - FR-5: Optional pose estimation
+    - NFR-1: Determinism (idempotent outputs)
+    - NFR-2: Performance (batch processing)
+
+Example:
+    >>> from w2t_bkin.dlc import run_dlc_inference_batch, DLCInferenceOptions
+    >>> from pathlib import Path
+    >>>
+    >>> videos = [Path("cam0.mp4"), Path("cam1.mp4")]
+    >>> model_config = Path("models/dlc_model/config.yaml")
+    >>> output_dir = Path("output/dlc")
+    >>>
+    >>> options = DLCInferenceOptions(gputouse=0, save_as_csv=False)
+    >>> results = run_dlc_inference_batch(videos, model_config, output_dir, options)
+    >>>
+    >>> for result in results:
+    ...     if result.success:
+    ...         print(f"Success: {result.h5_output_path}")
+    ...     else:
+    ...         print(f"Failed: {result.error_message}")
+"""
+
+from w2t_bkin.dlc.core import DLCInferenceError, auto_detect_gpu, predict_output_paths, run_dlc_inference_batch, validate_dlc_model
+from w2t_bkin.dlc.models import DLCInferenceOptions, DLCInferenceResult, DLCModelInfo
+
+__all__ = [
+    # Exception
+    "DLCInferenceError",
+    # Models
+    "DLCInferenceOptions",
+    "DLCInferenceResult",
+    "DLCModelInfo",
+    # Functions
+    "run_dlc_inference_batch",
+    "validate_dlc_model",
+    "predict_output_paths",
+    "auto_detect_gpu",
+]