fraclab-sdk 0.1.0__py3-none-any.whl

fraclab_sdk/runtime/snapshot_provider.py

@@ -0,0 +1 @@
+"""Snapshot data provider for algorithm runtime."""

fraclab_sdk/selection/__init__.py

@@ -0,0 +1,11 @@
+"""Selection management."""
+
+from fraclab_sdk.selection.model import SelectableDataset, SelectionModel
+from fraclab_sdk.selection.validate import ValidationError, validate_cardinality
+
+__all__ = [
+    "SelectableDataset",
+    "SelectionModel",
+    "ValidationError",
+    "validate_cardinality",
+]

fraclab_sdk/selection/model.py

@@ -0,0 +1,247 @@
+"""Selection model implementation."""
+
+from dataclasses import dataclass
+
+from fraclab_sdk.errors import DatasetKeyError
+from fraclab_sdk.models import DRS, DataSpec, DataSpecDataset, DataSpecItem, DRSDataset
+from fraclab_sdk.selection.validate import ValidationError, validate_cardinality
+from fraclab_sdk.snapshot.loader import SnapshotHandle
+
+
+@dataclass
+class SelectableDataset:
+    """Information about a selectable dataset."""
+
+    dataset_key: str
+    cardinality: str
+    total_items: int
+    resource_type: str | None = None
+    description: str | None = None
+
+
+class SelectionModel:
+    """Manages selection state for creating a run.
+
+    Selection uses snapshot item indices (integers).
+    Items are automatically sorted and deduplicated.
+    build_run_ds() produces a re-indexed DataSpec (0..N-1).
+    """
+
+    def __init__(
+        self,
+        snapshot: SnapshotHandle,
+        drs: DRS,
+    ) -> None:
+        """Initialize selection model.
+
+        Args:
+            snapshot: The snapshot handle.
+            drs: The data requirement specification (from algorithm).
+
+        Raises:
+            DatasetKeyError: If DRS requires a dataset not in snapshot.
+        """
+        self._snapshot = snapshot
+
+        # If DRS is empty, infer from snapshot dataspec so users can still select data.
+        if not drs.datasets:
+            inferred = [
+                DRSDataset(
+                    datasetKey=ds.datasetKey,
+                    resourceType=ds.resourceType,
+                    cardinality="many",
+                    description=ds.layout,
+                )
+                for ds in snapshot.dataspec.datasets
+            ]
+            self._drs = DRS(schemaVersion=drs.schemaVersion, datasets=inferred)
+        else:
+            self._drs = drs
+        self._selections: dict[str, list[int]] = {}
+
+        # Validate that all DRS dataset keys exist in snapshot
+        snapshot_keys = set(snapshot.dataspec.get_dataset_keys())
+        for drs_dataset in self._drs.datasets:
+            if drs_dataset.datasetKey not in snapshot_keys:
+                raise DatasetKeyError(
+                    dataset_key=drs_dataset.datasetKey,
+                    available_keys=list(snapshot_keys),
+                )
+            # Initialize empty selection
+            self._selections[drs_dataset.datasetKey] = []
+
+    @classmethod
+    def from_snapshot_and_drs(
+        cls,
+        snapshot: SnapshotHandle,
+        drs: DRS,
+    ) -> "SelectionModel":
+        """Create a SelectionModel from snapshot and DRS.
+
+        Args:
+            snapshot: The snapshot handle.
+            drs: The data requirement specification.
+
+        Returns:
+            Initialized SelectionModel.
+        """
+        return cls(snapshot=snapshot, drs=drs)
+
+    def get_selectable_datasets(self) -> list[SelectableDataset]:
+        """Get list of datasets that can be selected.
+
+        Returns:
+            List of SelectableDataset with metadata.
+        """
+        result = []
+        for drs_dataset in self._drs.datasets:
+            snapshot_dataset = self._snapshot.dataspec.get_dataset(drs_dataset.datasetKey)
+            if snapshot_dataset:
+                result.append(
+                    SelectableDataset(
+                        dataset_key=drs_dataset.datasetKey,
+                        cardinality=drs_dataset.cardinality,
+                        total_items=len(snapshot_dataset.items),
+                        resource_type=drs_dataset.resourceType,
+                        description=drs_dataset.description,
+                    )
+                )
+        return result
+
+    def set_selected(self, dataset_key: str, item_indices: list[int]) -> None:
+        """Set selected items for a dataset.
+
+        Items are automatically sorted (ascending) and deduplicated.
+
+        Args:
+            dataset_key: The dataset key.
+            item_indices: List of item indices to select.
+
+        Raises:
+            DatasetKeyError: If dataset_key is not in the selection.
+        """
+        if dataset_key not in self._selections:
+            raise DatasetKeyError(
+                dataset_key=dataset_key,
+                available_keys=list(self._selections.keys()),
+            )
+        # Sort and deduplicate
+        self._selections[dataset_key] = sorted(set(item_indices))
+
+    def get_selected(self, dataset_key: str) -> list[int]:
+        """Get selected item indices for a dataset.
+
+        Returns:
+            Sorted list of selected item indices.
+
+        Raises:
+            DatasetKeyError: If dataset_key is not in the selection.
+        """
+        if dataset_key not in self._selections:
+            raise DatasetKeyError(
+                dataset_key=dataset_key,
+                available_keys=list(self._selections.keys()),
+            )
+        return self._selections[dataset_key]
+
+    def validate(self) -> list[ValidationError]:
+        """Validate all selections against cardinality constraints.
+
+        Returns:
+            List of validation errors (empty if all valid).
+        """
+        errors = []
+        for drs_dataset in self._drs.datasets:
+            selected = self._selections.get(drs_dataset.datasetKey, [])
+            error = validate_cardinality(
+                dataset_key=drs_dataset.datasetKey,
+                cardinality=drs_dataset.cardinality,
+                selected_count=len(selected),
+            )
+            if error:
+                errors.append(error)
+        return errors
+
+    def is_valid(self) -> bool:
+        """Check if current selection is valid.
+
+        Returns:
+            True if all selections satisfy cardinality constraints.
+        """
+        return len(self.validate()) == 0
+
+    def _infer_layout(self, dataset_key: str) -> str | None:
+        """Infer layout from dataspec, manifest, or on-disk data."""
+        ds = self._snapshot.dataspec.get_dataset(dataset_key)
+        if ds and ds.layout:
+            return ds.layout
+
+        manifest_ds = self._snapshot.manifest.datasets.get(dataset_key)
+        if manifest_ds and getattr(manifest_ds, "layout", None):
+            return manifest_ds.layout
+
+        data_root = self._snapshot.manifest.dataRoot or "data"
+        base = self._snapshot.directory / data_root / dataset_key
+        if (base / "object.ndjson").exists():
+            return "object_ndjson_lines"
+        if (base / "parquet").exists():
+            return "frame_parquet_item_dirs"
+        return None
+
+    def build_run_ds(self) -> DataSpec:
+        """Build a run DataSpec from current selection.
+
+        Selected items are re-indexed to 0..N-1 (compact indices).
+        Each item includes sourceItemIndex for traceability.
+
+        Returns:
+            DataSpec with selected items, re-indexed.
+        """
+        datasets = []
+
+        for drs_dataset in self._drs.datasets:
+            snapshot_dataset = self._snapshot.dataspec.get_dataset(drs_dataset.datasetKey)
+            if not snapshot_dataset:
+                continue
+
+            selected_indices = self._selections.get(drs_dataset.datasetKey, [])
+
+            # Build re-indexed items
+            items = []
+            for snapshot_index in selected_indices:
+                if 0 <= snapshot_index < len(snapshot_dataset.items):
+                    original_item = snapshot_dataset.items[snapshot_index]
+                    # Create new item with sourceItemIndex for traceability
+                    new_item = DataSpecItem(
+                        owner=original_item.owner,
+                        resolutionParams=original_item.resolutionParams,
+                        range=original_item.range,
+                        sourceItemIndex=snapshot_index,
+                    )
+                    items.append(new_item)
+
+            datasets.append(
+                DataSpecDataset(
+                    datasetKey=snapshot_dataset.datasetKey,
+                    resourceType=snapshot_dataset.resourceType,
+                    layout=self._infer_layout(snapshot_dataset.datasetKey),
+                    items=items,
+                )
+            )
+
+        return DataSpec(
+            schemaVersion=self._snapshot.dataspec.schemaVersion,
+            datasets=datasets,
+        )
+
+    def get_selection_mapping(self, dataset_key: str) -> list[tuple[int, int]]:
+        """Get mapping from run index to snapshot index.
+
+        Args:
+            dataset_key: The dataset key.
+
+        Returns:
+            List of (run_index, snapshot_index) tuples.
+        """
+        selected = self.get_selected(dataset_key)
+        return list(enumerate(selected))

fraclab_sdk/selection/validate.py

@@ -0,0 +1,54 @@
+"""Selection validation implementation."""
+
+from dataclasses import dataclass
+
+
+@dataclass
+class ValidationError:
+    """Represents a validation error."""
+
+    dataset_key: str
+    message: str
+    error_type: str  # "cardinality", "missing_key", etc.
+
+
+def validate_cardinality(
+    dataset_key: str,
+    cardinality: str,
+    selected_count: int,
+) -> ValidationError | None:
+    """Validate selection against cardinality constraint.
+
+    Args:
+        dataset_key: The dataset key.
+        cardinality: The cardinality constraint ("one", "many", "zeroOrMany").
+        selected_count: Number of selected items.
+
+    Returns:
+        ValidationError if invalid, None if valid.
+    """
+    if cardinality == "one":
+        if selected_count != 1:
+            return ValidationError(
+                dataset_key=dataset_key,
+                message=f"Cardinality 'one' requires exactly 1 item, got {selected_count}",
+                error_type="cardinality",
+            )
+    elif cardinality == "many":
+        if selected_count < 1:
+            return ValidationError(
+                dataset_key=dataset_key,
+                message=f"Cardinality 'many' requires at least 1 item, got {selected_count}",
+                error_type="cardinality",
+            )
+    elif cardinality == "zeroOrMany":
+        # Always valid
+        pass
+    else:
+        return ValidationError(
+            dataset_key=dataset_key,
+            message=f"Unknown cardinality: {cardinality}",
+            error_type="unknown_cardinality",
+        )
+
+    return None

fraclab_sdk/snapshot/__init__.py

@@ -0,0 +1,12 @@
+"""Snapshot management."""
+
+from fraclab_sdk.snapshot.index import SnapshotIndex, SnapshotMeta
+from fraclab_sdk.snapshot.library import SnapshotLibrary
+from fraclab_sdk.snapshot.loader import SnapshotHandle
+
+__all__ = [
+    "SnapshotIndex",
+    "SnapshotLibrary",
+    "SnapshotHandle",
+    "SnapshotMeta",
+]

fraclab_sdk/snapshot/index.py

@@ -0,0 +1,94 @@
+"""Snapshot index management."""
+
+import json
+from dataclasses import dataclass, field
+from datetime import datetime
+from pathlib import Path
+
+from fraclab_sdk.utils.io import atomic_write_json
+
+
+@dataclass
+class SnapshotMeta:
+    """Metadata for an indexed snapshot."""
+
+    snapshot_id: str
+    bundle_id: str
+    created_at: str
+    description: str | None = None
+    imported_at: str = field(default_factory=lambda: datetime.now().isoformat())
+
+
+class SnapshotIndex:
+    """Manages the snapshot index file."""
+
+    def __init__(self, snapshots_dir: Path) -> None:
+        """Initialize snapshot index.
+
+        Args:
+            snapshots_dir: Directory containing snapshots.
+        """
+        self._snapshots_dir = snapshots_dir
+        self._index_path = snapshots_dir / "index.json"
+
+    def _load(self) -> dict[str, dict]:
+        """Load index from disk."""
+        if not self._index_path.exists():
+            return {}
+        return json.loads(self._index_path.read_text())
+
+    def _save(self, data: dict[str, dict]) -> None:
+        """Save index to disk."""
+        self._snapshots_dir.mkdir(parents=True, exist_ok=True)
+        atomic_write_json(self._index_path, data)
+
+    def add(self, meta: SnapshotMeta) -> None:
+        """Add a snapshot to the index."""
+        data = self._load()
+        data[meta.snapshot_id] = {
+            "snapshot_id": meta.snapshot_id,
+            "bundle_id": meta.bundle_id,
+            "created_at": meta.created_at,
+            "description": meta.description,
+            "imported_at": meta.imported_at,
+        }
+        self._save(data)
+
+    def remove(self, snapshot_id: str) -> None:
+        """Remove a snapshot from the index."""
+        data = self._load()
+        if snapshot_id in data:
+            del data[snapshot_id]
+            self._save(data)
+
+    def get(self, snapshot_id: str) -> SnapshotMeta | None:
+        """Get snapshot metadata by ID."""
+        data = self._load()
+        if snapshot_id not in data:
+            return None
+        entry = data[snapshot_id]
+        return SnapshotMeta(
+            snapshot_id=entry["snapshot_id"],
+            bundle_id=entry["bundle_id"],
+            created_at=entry["created_at"],
+            description=entry.get("description"),
+            imported_at=entry.get("imported_at", ""),
+        )
+
+    def list_all(self) -> list[SnapshotMeta]:
+        """List all indexed snapshots."""
+        data = self._load()
+        return [
+            SnapshotMeta(
+                snapshot_id=entry["snapshot_id"],
+                bundle_id=entry["bundle_id"],
+                created_at=entry["created_at"],
+                description=entry.get("description"),
+                imported_at=entry.get("imported_at", ""),
+            )
+            for entry in data.values()
+        ]
+
+    def contains(self, snapshot_id: str) -> bool:
+        """Check if a snapshot is in the index."""
+        return snapshot_id in self._load()

fraclab_sdk/snapshot/library.py

@@ -0,0 +1,205 @@
+"""Snapshot library implementation."""
+
+import shutil
+import zipfile
+from pathlib import Path
+
+from fraclab_sdk.config import SDKConfig
+from fraclab_sdk.errors import HashMismatchError, PathTraversalError, SnapshotError
+from fraclab_sdk.materialize.hash import compute_sha256
+from fraclab_sdk.models import BundleManifest
+from fraclab_sdk.snapshot.index import SnapshotIndex, SnapshotMeta
+from fraclab_sdk.snapshot.loader import SnapshotHandle
+
+
+def _is_safe_path(path: str) -> bool:
+    """Check if a path is safe (no traversal attacks)."""
+    if path.startswith("/") or path.startswith("\\"):
+        return False
+    if ".." in path:
+        return False
+    return not any(c in path for c in [":", "*", "?", '"', "<", ">", "|"])
+
+
+def _generate_snapshot_id(manifest_bytes: bytes) -> str:
+    """Generate snapshot ID from manifest content hash.
+
+    Uses SHA256 of manifest bytes, truncated to 16 chars for readability.
+    """
+    full_hash = compute_sha256(manifest_bytes)
+    return full_hash[:16]
+
+
+class SnapshotLibrary:
+    """Library for managing snapshots."""
+
+    def __init__(self, config: SDKConfig | None = None) -> None:
+        """Initialize snapshot library.
+
+        Args:
+            config: SDK configuration. If None, uses default.
+        """
+        self._config = config or SDKConfig()
+        self._index = SnapshotIndex(self._config.snapshots_dir)
+
+    def import_snapshot(self, path: Path) -> str:
+        """Import a snapshot from a directory or zip file.
+
+        Args:
+            path: Path to snapshot directory or zip file.
+
+        Returns:
+            The snapshot_id of the imported snapshot.
+
+        Raises:
+            SnapshotError: If import fails.
+            HashMismatchError: If hash verification fails.
+            PathTraversalError: If zip contains unsafe paths.
+        """
+        path = path.resolve()
+        if not path.exists():
+            raise SnapshotError(f"Path does not exist: {path}")
+
+        if path.is_file() and path.suffix == ".zip":
+            return self._import_from_zip(path)
+        elif path.is_dir():
+            return self._import_from_dir(path)
+        else:
+            raise SnapshotError(f"Path must be a directory or .zip file: {path}")
+
+    def _import_from_zip(self, zip_path: Path) -> str:
+        """Import snapshot from zip file."""
+        import tempfile
+
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            tmp_path = Path(tmp_dir)
+
+            with zipfile.ZipFile(zip_path) as zf:
+                # Security check: verify all paths are safe
+                for name in zf.namelist():
+                    if not _is_safe_path(name):
+                        raise PathTraversalError(name)
+                zf.extractall(tmp_path)
+
+            # Find the actual snapshot root (may be in a subdirectory)
+            snapshot_root = self._find_snapshot_root(tmp_path)
+            return self._import_from_dir(snapshot_root)
+
+    def _find_snapshot_root(self, path: Path) -> Path:
+        """Find the snapshot root directory (contains manifest.json)."""
+        if (path / "manifest.json").exists():
+            return path
+
+        # Check one level of subdirectories
+        for subdir in path.iterdir():
+            if subdir.is_dir() and (subdir / "manifest.json").exists():
+                return subdir
+
+        raise SnapshotError(f"No manifest.json found in {path}")
+
+    def _import_from_dir(self, source_dir: Path) -> str:
+        """Import snapshot from directory."""
+        # Validate manifest exists
+        manifest_path = source_dir / "manifest.json"
+        if not manifest_path.exists():
+            raise SnapshotError(f"manifest.json not found in {source_dir}")
+
+        # Parse manifest and get file paths
+        manifest_bytes = manifest_path.read_bytes()
+        manifest = BundleManifest.model_validate_json(manifest_bytes.decode())
+
+        ds_path = source_dir / manifest.specFiles.dsPath
+        drs_path = source_dir / manifest.specFiles.drsPath
+        data_dir = source_dir / manifest.dataRoot
+
+        # Validate required files exist
+        if not ds_path.exists():
+            raise SnapshotError(f"{manifest.specFiles.dsPath} not found in {source_dir}")
+        if not drs_path.exists():
+            raise SnapshotError(
+                f"{manifest.specFiles.drsPath} not found (REQUIRED): {drs_path}"
+            )
+        if not data_dir.exists():
+            raise SnapshotError(f"{manifest.dataRoot}/ directory not found in {source_dir}")
+
+        # Verify hashes on raw bytes
+        ds_bytes = ds_path.read_bytes()
+        ds_hash = compute_sha256(ds_bytes)
+        if ds_hash != manifest.specFiles.dsSha256:
+            raise HashMismatchError(
+                manifest.specFiles.dsPath, manifest.specFiles.dsSha256, ds_hash
+            )
+
+        drs_bytes = drs_path.read_bytes()
+        drs_hash = compute_sha256(drs_bytes)
+        if drs_hash != manifest.specFiles.drsSha256:
+            raise HashMismatchError(
+                manifest.specFiles.drsPath, manifest.specFiles.drsSha256, drs_hash
+            )
+
+        # Generate snapshot_id from manifest hash
+        snapshot_id = _generate_snapshot_id(manifest_bytes)
+
+        # Create target directory
+        self._config.ensure_dirs()
+        target_dir = self._config.snapshots_dir / snapshot_id
+
+        if target_dir.exists():
+            # Already imported
+            return snapshot_id
+
+        # Copy to library
+        shutil.copytree(source_dir, target_dir)
+
+        # Add to index
+        self._index.add(
+            SnapshotMeta(
+                snapshot_id=snapshot_id,
+                bundle_id=snapshot_id,
+                created_at=str(manifest.createdAtUs),
+                description=None,
+            )
+        )
+
+        return snapshot_id
+
+    def list_snapshots(self) -> list[SnapshotMeta]:
+        """List all imported snapshots.
+
+        Returns:
+            List of snapshot metadata.
+        """
+        return self._index.list_all()
+
+    def get_snapshot(self, snapshot_id: str) -> SnapshotHandle:
+        """Get a handle to a snapshot.
+
+        Args:
+            snapshot_id: The snapshot ID.
+
+        Returns:
+            SnapshotHandle for accessing snapshot contents.
+
+        Raises:
+            SnapshotError: If snapshot not found.
+        """
+        snapshot_dir = self._config.snapshots_dir / snapshot_id
+        if not snapshot_dir.exists():
+            raise SnapshotError(f"Snapshot not found: {snapshot_id}")
+        return SnapshotHandle(snapshot_dir)
+
+    def delete_snapshot(self, snapshot_id: str) -> None:
+        """Delete a snapshot from the library.
+
+        Args:
+            snapshot_id: The snapshot ID to delete.
+
+        Raises:
+            SnapshotError: If snapshot not found.
+        """
+        snapshot_dir = self._config.snapshots_dir / snapshot_id
+        if not snapshot_dir.exists():
+            raise SnapshotError(f"Snapshot not found: {snapshot_id}")
+
+        shutil.rmtree(snapshot_dir)
+        self._index.remove(snapshot_id)