bead 0.1.0__py3-none-any.whl

bead/data/validation.py

@@ -0,0 +1,349 @@
+"""Validation utilities for data integrity checks.
+
+This module provides validation functions beyond Pydantic's built-in validation,
+including file validation, reference validation, and provenance chain validation.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import get_type_hints
+from uuid import UUID
+
+from pydantic import BaseModel, Field, ValidationError
+
+from bead.data.metadata import MetadataTracker
+
+
+class ValidationReport(BaseModel):
+    """Report of validation results.
+
+    A lightweight model for collecting and reporting validation results,
+    including errors, warnings, and statistics about validated objects.
+
+    Attributes
+    ----------
+    valid : bool
+        Overall validation status (False if any errors)
+    errors : list[str]
+        List of error messages (default: empty list)
+    warnings : list[str]
+        List of warning messages (default: empty list)
+    object_count : int
+        Number of objects validated (default: 0)
+
+    Examples
+    --------
+    >>> report = ValidationReport(valid=True)
+    >>> report.add_error("Invalid field")
+    >>> report.valid
+    False
+    >>> report.has_errors()
+    True
+    >>> len(report.errors)
+    1
+    """
+
+    valid: bool
+    errors: list[str] = Field(default_factory=list)
+    warnings: list[str] = Field(default_factory=list)
+    object_count: int = 0
+
+    def add_error(self, message: str) -> None:
+        """Add an error message and set valid to False.
+
+        Parameters
+        ----------
+        message
+            Error message to add.
+
+        Examples
+        --------
+        >>> report = ValidationReport(valid=True)
+        >>> report.add_error("Something went wrong")
+        >>> report.valid
+        False
+        >>> "Something went wrong" in report.errors
+        True
+        """
+        self.errors.append(message)
+        self.valid = False
+
+    def add_warning(self, message: str) -> None:
+        """Add a warning message.
+
+        Warnings do not affect the valid status.
+
+        Parameters
+        ----------
+        message
+            Warning message to add.
+
+        Examples
+        --------
+        >>> report = ValidationReport(valid=True)
+        >>> report.add_warning("This might be an issue")
+        >>> report.valid
+        True
+        >>> report.has_warnings()
+        True
+        """
+        self.warnings.append(message)
+
+    def has_errors(self) -> bool:
+        """Check if report has any errors.
+
+        Returns
+        -------
+        bool
+            True if errors list is non-empty
+
+        Examples
+        --------
+        >>> report = ValidationReport(valid=True)
+        >>> report.has_errors()
+        False
+        >>> report.add_error("error")
+        >>> report.has_errors()
+        True
+        """
+        return len(self.errors) > 0
+
+    def has_warnings(self) -> bool:
+        """Check if report has any warnings.
+
+        Returns
+        -------
+        bool
+            True if warnings list is non-empty
+
+        Examples
+        --------
+        >>> report = ValidationReport(valid=True)
+        >>> report.has_warnings()
+        False
+        >>> report.add_warning("warning")
+        >>> report.has_warnings()
+        True
+        """
+        return len(self.warnings) > 0
+
+
+def validate_jsonlines_file(
+    path: Path, model_class: type[BaseModel], strict: bool = True
+) -> ValidationReport:
+    """Validate JSONLines file against Pydantic model schema.
+
+    Reads and validates each line in a JSONLines file against the provided
+    model class. Empty lines are skipped.
+
+    Parameters
+    ----------
+    path
+        Path to JSONLines file to validate.
+    model_class
+        Pydantic model class to validate against.
+    strict
+        If True, stop at first error. If False, collect all errors (default: True).
+
+    Returns
+    -------
+    ValidationReport
+        Validation report with results
+
+    Examples
+    --------
+    >>> from pathlib import Path
+    >>> from bead.data.base import BeadBaseModel
+    >>> class TestModel(BeadBaseModel):
+    ...     name: str
+    >>> # validate file
+    >>> report = validate_jsonlines_file(
+    ...     Path("data.jsonl"), TestModel
+    ... )  # doctest: +SKIP
+    >>> report.valid
+    True
+    """
+    report = ValidationReport(valid=True)
+
+    # check if file exists
+    if not path.exists():
+        report.add_error(f"File not found: {path}")
+        return report
+
+    try:
+        # try to read the file
+        with path.open("r", encoding="utf-8") as f:
+            for line_num, line in enumerate(f, start=1):
+                line = line.strip()
+                if not line:  # skip empty lines
+                    continue
+
+                try:
+                    # try to parse and validate
+                    model_class.model_validate_json(line)
+                    report.object_count += 1
+                except ValidationError as e:
+                    error_msg = f"Line {line_num}: Validation error - {e}"
+                    report.add_error(error_msg)
+                    if strict:
+                        return report
+                except Exception as e:
+                    error_msg = f"Line {line_num}: Parse error - {e}"
+                    report.add_error(error_msg)
+                    if strict:
+                        return report
+
+    except OSError as e:
+        report.add_error(f"Failed to read file: {e}")
+
+    return report
+
+
+def validate_uuid_references(
+    objects: list[BaseModel], reference_pool: dict[UUID, BaseModel]
+) -> ValidationReport:
+    """Validate that UUID references point to existing objects.
+
+    Checks all UUID fields in objects to ensure they reference valid objects
+    in the reference pool. Supports both single UUID fields and list[UUID] fields.
+
+    Parameters
+    ----------
+    objects
+        List of objects to validate.
+    reference_pool
+        Dictionary of valid UUIDs to objects.
+
+    Returns
+    -------
+    ValidationReport
+        Validation report with results
+
+    Examples
+    --------
+    >>> from uuid import uuid4
+    >>> from bead.data.base import BeadBaseModel
+    >>> class Item(BeadBaseModel):
+    ...     name: str
+    >>> items = [Item(name="test")]
+    >>> pool = {items[0].id: items[0]}
+    >>> report = validate_uuid_references(items, pool)
+    >>> report.valid
+    True
+    """
+    report = ValidationReport(valid=True)
+    report.object_count = len(objects)
+
+    for obj in objects:
+        # get type hints for the object
+        try:
+            type_hints = get_type_hints(type(obj))
+        except Exception:
+            # if we can't get type hints, skip this object
+            continue
+
+        # check each field
+        for field_name, field_type in type_hints.items():
+            # skip 'id' field; it's the object's own ID, not a reference
+            if field_name == "id":
+                continue
+
+            # convert type to string for checking
+            type_str = str(field_type)
+
+            # check if field contains UUID
+            if "UUID" not in type_str:
+                continue
+
+            # get field value
+            try:
+                field_value = getattr(obj, field_name)
+            except AttributeError:
+                continue
+
+            # check if it's a list of UUIDs
+            if "list" in type_str.lower() or "List" in type_str:
+                if isinstance(field_value, list):
+                    for item in field_value:  # pyright: ignore[reportUnknownVariableType]
+                        if not isinstance(item, UUID):
+                            continue
+                        if item not in reference_pool:
+                            # get object ID for error message
+                            obj_id = getattr(obj, "id", "unknown")
+                            report.add_error(
+                                f"Object {obj_id}: "
+                                f"Field '{field_name}' references "
+                                f"missing UUID {item}"
+                            )
+            # single UUID field
+            elif isinstance(field_value, UUID):
+                if field_value not in reference_pool:
+                    # get object ID for error message
+                    obj_id = getattr(obj, "id", "unknown")
+                    report.add_error(
+                        f"Object {obj_id}: "
+                        f"Field '{field_name}' references "
+                        f"missing UUID {field_value}"
+                    )
+
+    return report
+
+
+def validate_provenance_chain(
+    metadata: MetadataTracker, repository: dict[UUID, BaseModel]
+) -> ValidationReport:
+    """Validate provenance chain references are valid.
+
+    Checks that all parent_id references in the provenance chain exist in the
+    repository and that parent_type matches the actual type.
+
+    Parameters
+    ----------
+    metadata
+        Metadata tracker with provenance chain to validate.
+    repository
+        Dictionary of valid UUIDs to objects.
+
+    Returns
+    -------
+    ValidationReport
+        Validation report with results
+
+    Examples
+    --------
+    >>> from uuid import uuid4
+    >>> from bead.data.base import BeadBaseModel
+    >>> from bead.data.metadata import MetadataTracker
+    >>> class Template(BeadBaseModel):
+    ...     name: str
+    >>> template = Template(name="test")
+    >>> metadata = MetadataTracker()
+    >>> metadata.add_provenance(template.id, "Template", "filled_from")
+    >>> repo = {template.id: template}
+    >>> report = validate_provenance_chain(metadata, repo)
+    >>> report.valid
+    True
+    """
+    report = ValidationReport(valid=True)
+    report.object_count = len(metadata.provenance)
+
+    for record in metadata.provenance:
+        # check if parent exists
+        if record.parent_id not in repository:
+            report.add_error(
+                f"Provenance record references missing parent: {record.parent_id}"
+            )
+            continue
+
+        # check if parent_type matches actual type
+        parent_obj = repository[record.parent_id]
+        actual_type = type(parent_obj).__name__
+
+        if record.parent_type != actual_type:
+            report.add_error(
+                f"Provenance record for {record.parent_id}: "
+                f"Expected type '{record.parent_type}', got '{actual_type}'"
+            )
+
+    return report

bead/data_collection/__init__.py

@@ -0,0 +1,11 @@
+"""Data collection infrastructure for human experiments."""
+
+from bead.data_collection.jatos import JATOSDataCollector
+from bead.data_collection.merger import DataMerger
+from bead.data_collection.prolific import ProlificDataCollector
+
+__all__ = [
+    "JATOSDataCollector",
+    "ProlificDataCollector",
+    "DataMerger",
+]

bead/data_collection/jatos.py

@@ -0,0 +1,223 @@
+"""JATOS data collection for model training.
+
+This module provides the JATOSDataCollector class for downloading experimental
+results from JATOS servers. It wraps the existing JATOSClient and adds
+functionality for:
+- Downloading all results for a study
+- Filtering by component and worker type
+- Adding metadata (timestamps, etc.)
+- Saving to JSONLines format
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from bead.data.base import JsonValue
+from bead.data.timestamps import now_iso8601
+from bead.deployment.jatos.api import JATOSClient
+
+
+class JATOSDataCollector:
+    """Collects experimental data from JATOS API.
+
+    This class wraps the existing JATOSClient to provide data collection
+    functionality specifically for model training. It downloads results,
+    adds metadata, and saves in JSONLines format.
+
+    Parameters
+    ----------
+    base_url : str
+        JATOS instance URL (e.g., https://jatos.example.com).
+    api_token : str
+        API authentication token.
+    study_id : int
+        JATOS study ID to collect data from.
+
+    Attributes
+    ----------
+    study_id : int
+        JATOS study ID to collect data from.
+    client : JATOSClient
+        Underlying JATOS API client.
+
+    Examples
+    --------
+    Create a collector and download results::
+
+        collector = JATOSDataCollector(
+            base_url="https://jatos.example.com",
+            api_token="my-token",
+            study_id=123
+        )
+        results = collector.download_results(Path("results.jsonl"))
+    """
+
+    def __init__(
+        self,
+        base_url: str,
+        api_token: str,
+        study_id: int,
+    ) -> None:
+        self.study_id = study_id
+        self.client = JATOSClient(base_url, api_token)
+
+    def download_results(
+        self,
+        output_path: Path,
+        component_id: int | None = None,
+        worker_type: str | None = None,
+    ) -> list[dict[str, JsonValue]]:
+        """Download all results for the study.
+
+        Downloads results from JATOS, optionally filtering by component ID
+        and worker type. Each result is enriched with download timestamp
+        metadata and saved to a JSONLines file (one result per line).
+
+        Parameters
+        ----------
+        output_path : Path
+            Path to save results (JSONLines format).
+        component_id : int | None
+            Filter by component ID (optional).
+        worker_type : str | None
+            Filter by worker type (optional).
+
+        Returns
+        -------
+        list[dict[str, JsonValue]]
+            Downloaded results with metadata.
+
+        Raises
+        ------
+        requests.HTTPError
+            If the API request fails.
+
+        Examples
+        --------
+        Download all results::
+
+            results = collector.download_results(Path("results.jsonl"))
+
+        Download with filters::
+
+            results = collector.download_results(
+                Path("results.jsonl"),
+                component_id=1,
+                worker_type="Prolific"
+            )
+        """
+        # Get result IDs from JATOS API
+        result_ids = self.client.get_results(self.study_id)
+
+        results: list[dict[str, JsonValue]] = []
+
+        # Download each result with metadata
+        for result_id in result_ids:
+            result = self._download_single_result(result_id)
+
+            # Apply filters
+            if component_id is not None:
+                result_component_id = result.get("metadata", {}).get("componentId")
+                if result_component_id != component_id:
+                    continue
+
+            if worker_type is not None:
+                result_worker_type = result.get("metadata", {}).get("workerType")
+                if result_worker_type != worker_type:
+                    continue
+
+            results.append(result)
+
+        # Save to JSONLines file (one result per line)
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        with open(output_path, "w") as f:
+            for result in results:
+                f.write(json.dumps(result) + "\n")
+
+        return results
+
+    def _download_single_result(self, result_id: int) -> dict[str, JsonValue]:
+        """Download a single result with metadata.
+
+        Parameters
+        ----------
+        result_id : int
+            Result ID to download.
+
+        Returns
+        -------
+        dict[str, JsonValue]
+            Result data with metadata and download timestamp.
+
+        Raises
+        ------
+        requests.HTTPError
+            If the API request fails.
+        """
+        # Get result data
+        data_url = f"{self.client.base_url}/api/v1/results/{result_id}/data"
+        data_response = self.client.session.get(data_url)
+        data_response.raise_for_status()
+
+        # Get result metadata
+        meta_url = f"{self.client.base_url}/api/v1/results/{result_id}"
+        meta_response = self.client.session.get(meta_url)
+        meta_response.raise_for_status()
+
+        metadata = meta_response.json()
+
+        return {
+            "result_id": result_id,
+            "data": data_response.json(),
+            "metadata": metadata,
+            "download_timestamp": now_iso8601().isoformat(),
+        }
+
+    def get_study_info(self) -> dict[str, JsonValue]:
+        """Get study information.
+
+        Delegates to the underlying JATOSClient.
+
+        Returns
+        -------
+        dict[str, JsonValue]
+            Study details dictionary.
+
+        Raises
+        ------
+        requests.HTTPError
+            If the API request fails.
+
+        Examples
+        --------
+        ::
+
+            info = collector.get_study_info()
+            print(info["title"])
+        """
+        return self.client.get_study(self.study_id)
+
+    def get_result_count(self) -> int:
+        """Get count of results.
+
+        Returns
+        -------
+        int
+            Number of results available for the study.
+
+        Raises
+        ------
+        requests.HTTPError
+            If the API request fails.
+
+        Examples
+        --------
+        ::
+
+            count = collector.get_result_count()
+            print(f"Found {count} results")
+        """
+        result_ids = self.client.get_results(self.study_id)
+        return len(result_ids)