fraclab-sdk 0.1.0__py3-none-any.whl

fraclab_sdk/errors.py

@@ -0,0 +1,124 @@
+"""SDK error definitions."""
+
+from enum import IntEnum
+
+
+class ExitCode(IntEnum):
+    """CLI exit codes for scripting/CI integration."""
+
+    SUCCESS = 0
+    GENERAL_ERROR = 1
+    INPUT_ERROR = 2  # Input/parameter errors (validation, path not found)
+    RUN_FAILED = 3  # Algorithm execution failed
+    TIMEOUT = 4  # Execution timed out
+    INTERNAL_ERROR = 5  # Unexpected internal error (bug)
+
+
+class FraclabError(Exception):
+    """Base exception for all Fraclab SDK errors.
+
+    Attributes:
+        exit_code: Recommended CLI exit code for this error type.
+    """
+
+    exit_code: ExitCode = ExitCode.GENERAL_ERROR
+
+
+class SnapshotError(FraclabError):
+    """Error related to snapshot operations."""
+
+    exit_code = ExitCode.INPUT_ERROR
+
+
+class AlgorithmError(FraclabError):
+    """Error related to algorithm operations."""
+
+    exit_code = ExitCode.INPUT_ERROR
+
+
+class SelectionError(FraclabError):
+    """Error related to selection operations."""
+
+    exit_code = ExitCode.INPUT_ERROR
+
+
+class MaterializeError(FraclabError):
+    """Error related to materialization operations."""
+
+    exit_code = ExitCode.INTERNAL_ERROR
+
+
+class RunError(FraclabError):
+    """Error related to run execution."""
+
+    exit_code = ExitCode.RUN_FAILED
+
+
+class TimeoutError(RunError):
+    """Error when run execution times out."""
+
+    exit_code = ExitCode.TIMEOUT
+
+
+class ResultError(FraclabError):
+    """Error related to result reading."""
+
+    exit_code = ExitCode.INPUT_ERROR
+
+
+class HashMismatchError(SnapshotError):
+    """Error when file hash doesn't match expected hash."""
+
+    def __init__(self, file_name: str, expected: str, actual: str) -> None:
+        self.file_name = file_name
+        self.expected = expected
+        self.actual = actual
+        super().__init__(
+            f"Hash mismatch for {file_name}: expected {expected[:16]}..., got {actual[:16]}..."
+        )
+
+
+class PathTraversalError(FraclabError):
+    """Error when a path traversal attempt is detected."""
+
+    def __init__(self, path: str) -> None:
+        self.path = path
+        super().__init__(f"Path traversal detected: {path}")
+
+
+class DatasetKeyError(SelectionError):
+    """Error when a required dataset key is not found."""
+
+    def __init__(self, dataset_key: str, available_keys: list[str]) -> None:
+        self.dataset_key = dataset_key
+        self.available_keys = available_keys
+        super().__init__(
+            f"Dataset key '{dataset_key}' not found. Available: {available_keys}"
+        )
+
+
+class CardinalityError(SelectionError):
+    """Error when selection violates cardinality constraints."""
+
+    def __init__(
+        self, dataset_key: str, cardinality: str, selected_count: int
+    ) -> None:
+        self.dataset_key = dataset_key
+        self.cardinality = cardinality
+        self.selected_count = selected_count
+        super().__init__(
+            f"Cardinality violation for '{dataset_key}': "
+            f"cardinality={cardinality}, selected={selected_count}"
+        )
+
+
+class OutputContainmentError(RunError):
+    """Error when algorithm attempts to write outside output directory."""
+
+    def __init__(self, attempted_path: str, output_dir: str) -> None:
+        self.attempted_path = attempted_path
+        self.output_dir = output_dir
+        super().__init__(
+            f"Output containment violation: attempted to write to {attempted_path}, "
+            f"but must be under {output_dir}"
+        )

fraclab_sdk/materialize/__init__.py

@@ -0,0 +1,8 @@
+"""Materialization management."""
+
+from fraclab_sdk.materialize.materializer import Materializer, MaterializeResult
+
+__all__ = [
+    "Materializer",
+    "MaterializeResult",
+]

fraclab_sdk/materialize/fsops.py

@@ -0,0 +1,125 @@
+"""File system operations for materialization."""
+
+import os
+import shutil
+from pathlib import Path
+
+
+def copy_file_smart(src: Path, dst: Path) -> str:
+    """Copy a file using hardlink > symlink > copy fallback strategy.
+
+    Args:
+        src: Source file path.
+        dst: Destination file path.
+
+    Returns:
+        Strategy used: "hardlink", "symlink", or "copy".
+
+    Raises:
+        FileNotFoundError: If source file doesn't exist.
+        OSError: If all copy strategies fail.
+    """
+    if not src.exists():
+        raise FileNotFoundError(f"Source file not found: {src}")
+
+    # Ensure parent directory exists
+    dst.parent.mkdir(parents=True, exist_ok=True)
+
+    # Try hardlink first
+    try:
+        os.link(src, dst)
+        return "hardlink"
+    except OSError:
+        pass
+
+    # Try symlink
+    try:
+        os.symlink(src.resolve(), dst)
+        return "symlink"
+    except OSError:
+        pass
+
+    # Fall back to copy
+    shutil.copy2(src, dst)
+    return "copy"
+
+
+def copy_directory_smart(src_dir: Path, dst_dir: Path) -> dict[str, int]:
+    """Copy directory contents using smart file copy strategy.
+
+    Each file in the source directory (including nested) is copied
+    using the hardlink > symlink > copy fallback strategy.
+
+    Args:
+        src_dir: Source directory path.
+        dst_dir: Destination directory path.
+
+    Returns:
+        Dict with counts: {"hardlink": N, "symlink": N, "copy": N}
+
+    Raises:
+        FileNotFoundError: If source directory doesn't exist.
+    """
+    if not src_dir.exists():
+        raise FileNotFoundError(f"Source directory not found: {src_dir}")
+
+    counts = {"hardlink": 0, "symlink": 0, "copy": 0}
+
+    for src_file in src_dir.rglob("*"):
+        if src_file.is_file():
+            rel_path = src_file.relative_to(src_dir)
+            dst_file = dst_dir / rel_path
+            strategy = copy_file_smart(src_file, dst_file)
+            counts[strategy] += 1
+
+    return counts
+
+
+def extract_ndjson_lines(
+    src_path: Path,
+    dst_path: Path,
+    line_indices: list[int],
+) -> int:
+    """Extract specific lines from ndjson file and write to new file.
+
+    Lines are written in the order of line_indices (which should be sorted).
+    Output file has contiguous line numbers (0..N-1).
+
+    Args:
+        src_path: Source ndjson file path.
+        dst_path: Destination ndjson file path.
+        line_indices: List of 0-based line indices to extract (must be sorted).
+
+    Returns:
+        Number of lines written.
+
+    Raises:
+        FileNotFoundError: If source file doesn't exist.
+        IndexError: If line index is out of range.
+    """
+    if not src_path.exists():
+        raise FileNotFoundError(f"Source file not found: {src_path}")
+
+    # Ensure parent directory exists
+    dst_path.parent.mkdir(parents=True, exist_ok=True)
+
+    # Read source and extract lines
+    with src_path.open("r", encoding="utf-8") as f:
+        all_lines = f.readlines()
+
+    # Validate indices and extract
+    extracted = []
+    for idx in line_indices:
+        if idx < 0 or idx >= len(all_lines):
+            raise IndexError(f"Line index {idx} out of range (0-{len(all_lines)-1})")
+        extracted.append(all_lines[idx])
+
+    # Write extracted lines
+    with dst_path.open("w", encoding="utf-8") as f:
+        for line in extracted:
+            # Ensure line ends with newline
+            if not line.endswith("\n"):
+                line += "\n"
+            f.write(line)
+
+    return len(extracted)

fraclab_sdk/materialize/hash.py

@@ -0,0 +1,28 @@
+"""Hash computation utilities."""
+
+import hashlib
+from pathlib import Path
+
+
+def compute_sha256(data: bytes) -> str:
+    """Compute SHA256 hash of bytes data.
+
+    Args:
+        data: Bytes to hash.
+
+    Returns:
+        Hex-encoded SHA256 hash string.
+    """
+    return hashlib.sha256(data).hexdigest()
+
+
+def compute_file_sha256(path: Path) -> str:
+    """Compute SHA256 hash of a file.
+
+    Args:
+        path: Path to file.
+
+    Returns:
+        Hex-encoded SHA256 hash string.
+    """
+    return compute_sha256(path.read_bytes())

fraclab_sdk/materialize/materializer.py

@@ -0,0 +1,241 @@
+"""Materializer implementation."""
+
+import json
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from fraclab_sdk.errors import MaterializeError
+from fraclab_sdk.materialize.fsops import copy_directory_smart, extract_ndjson_lines
+from fraclab_sdk.materialize.hash import compute_sha256
+from fraclab_sdk.models import DRS, DataSpec
+from fraclab_sdk.snapshot.loader import SnapshotHandle
+
+
+@dataclass
+class MaterializeResult:
+    """Result of materialization."""
+
+    input_dir: Path
+    ds_sha256: str
+    drs_sha256: str
+    copy_stats: dict[str, dict[str, int]]  # dataset_key -> {hardlink, symlink, copy}
+
+
+class Materializer:
+    """Materializes run input from snapshot and selection."""
+
+    def materialize(
+        self,
+        run_dir: Path,
+        snapshot: SnapshotHandle,
+        run_ds: DataSpec,
+        drs: DRS,
+        params: dict[str, Any],
+        run_context: dict[str, Any],
+    ) -> MaterializeResult:
+        """Materialize run input directory.
+
+        Creates runs/<run_id>/input/ with:
+        - manifest.json (with sha256 hashes)
+        - ds.json (run subset, re-indexed)
+        - drs.json (from algorithm)
+        - params.json
+        - run_context.json
+        - data/ (layout-aware materialization)
+
+        Args:
+            run_dir: The run directory (will create input/ subdirectory).
+            snapshot: Source snapshot handle.
+            run_ds: Run DataSpec (re-indexed from selection).
+            drs: Algorithm's DRS.
+            params: Algorithm parameters.
+            run_context: Run context metadata.
+
+        Returns:
+            MaterializeResult with paths and hashes.
+
+        Raises:
+            MaterializeError: If materialization fails.
+        """
+        input_dir = run_dir / "input"
+        input_dir.mkdir(parents=True, exist_ok=True)
+
+        # Write ds.json and compute hash
+        ds_bytes = self._write_json(input_dir / "ds.json", run_ds.model_dump())
+        ds_sha256 = compute_sha256(ds_bytes)
+
+        # Write drs.json and compute hash
+        drs_bytes = self._write_json(input_dir / "drs.json", drs.model_dump())
+        drs_sha256 = compute_sha256(drs_bytes)
+
+        # Write params.json
+        self._write_json(input_dir / "params.json", params)
+
+        # Write run_context.json
+        self._write_json(input_dir / "run_context.json", run_context)
+
+        # Materialize data
+        copy_stats = self._materialize_data(input_dir, snapshot, run_ds)
+
+        # Write manifest.json (last, with computed hashes)
+        # Build datasets entry for manifest
+        datasets_manifest: dict[str, dict] = {}
+        for dataset in run_ds.datasets:
+            datasets_manifest[dataset.datasetKey] = {
+                "layout": dataset.layout,
+                "count": len(dataset.items),
+            }
+
+        manifest = {
+            "bundleVersion": "1.0.0",
+            "createdAtUs": int(time.time() * 1_000_000),
+            "specFiles": {
+                "dsPath": "ds.json",
+                "drsPath": "drs.json",
+                "dsSha256": ds_sha256,
+                "drsSha256": drs_sha256,
+            },
+            "dataRoot": "data",
+            "datasets": datasets_manifest,
+        }
+        self._write_json(input_dir / "manifest.json", manifest)
+
+        return MaterializeResult(
+            input_dir=input_dir,
+            ds_sha256=ds_sha256,
+            drs_sha256=drs_sha256,
+            copy_stats=copy_stats,
+        )
+
+    def _write_json(self, path: Path, data: Any) -> bytes:
+        """Write JSON file and return bytes written.
+
+        Args:
+            path: File path to write.
+            data: Data to serialize as JSON.
+
+        Returns:
+            Bytes that were written.
+        """
+        content = json.dumps(data, indent=2, ensure_ascii=False)
+        content_bytes = content.encode("utf-8")
+        path.write_bytes(content_bytes)
+        return content_bytes
+
+    def _materialize_data(
+        self,
+        input_dir: Path,
+        snapshot: SnapshotHandle,
+        run_ds: DataSpec,
+    ) -> dict[str, dict[str, int]]:
+        """Materialize data directory with layout-aware copying.
+
+        Args:
+            input_dir: The input directory.
+            snapshot: Source snapshot.
+            run_ds: Run DataSpec with re-indexed items.
+
+        Returns:
+            Copy stats per dataset.
+        """
+        data_dir = input_dir / "data"
+        copy_stats: dict[str, dict[str, int]] = {}
+
+        for dataset in run_ds.datasets:
+            dataset_key = dataset.datasetKey
+            layout = dataset.layout
+
+            if layout == "frame_parquet_item_dirs":
+                stats = self._materialize_parquet(
+                    data_dir, snapshot, dataset_key, dataset.items
+                )
+                copy_stats[dataset_key] = stats
+            elif layout == "object_ndjson_lines":
+                self._materialize_ndjson(
+                    data_dir, snapshot, dataset_key, dataset.items
+                )
+                copy_stats[dataset_key] = {"ndjson_lines": len(dataset.items)}
+            else:
+                raise MaterializeError(f"Unknown layout: {layout}")
+
+        return copy_stats
+
+    def _materialize_parquet(
+        self,
+        data_dir: Path,
+        snapshot: SnapshotHandle,
+        dataset_key: str,
+        items: list,
+    ) -> dict[str, int]:
+        """Materialize parquet item directories with re-indexing.
+
+        Source: snapshot/data/<datasetKey>/parquet/item-<snapshot_index:05d>/
+        Target: run/input/data/<datasetKey>/parquet/item-<run_index:05d>/
+
+        Args:
+            data_dir: Target data directory.
+            snapshot: Source snapshot.
+            dataset_key: Dataset key.
+            items: List of DataSpecItem (with sourceItemIndex).
+
+        Returns:
+            Copy stats {hardlink, symlink, copy}.
+        """
+        total_stats = {"hardlink": 0, "symlink": 0, "copy": 0}
+
+        for run_index, item in enumerate(items):
+            snapshot_index = item.sourceItemIndex
+            if snapshot_index is None:
+                raise MaterializeError(
+                    f"Item at run index {run_index} missing sourceItemIndex"
+                )
+
+            src_dir = snapshot.get_item_dir(dataset_key, snapshot_index)
+            dst_dir = data_dir / dataset_key / "parquet" / f"item-{run_index:05d}"
+
+            if not src_dir.exists():
+                raise MaterializeError(f"Source item directory not found: {src_dir}")
+
+            stats = copy_directory_smart(src_dir, dst_dir)
+            for key in total_stats:
+                total_stats[key] += stats[key]
+
+        return total_stats
+
+    def _materialize_ndjson(
+        self,
+        data_dir: Path,
+        snapshot: SnapshotHandle,
+        dataset_key: str,
+        items: list,
+    ) -> None:
+        """Materialize ndjson by extracting selected lines.
+
+        Extracts lines by snapshot index and writes contiguously.
+        Run item 0 = line 0, run item 1 = line 1, etc.
+
+        Args:
+            data_dir: Target data directory.
+            snapshot: Source snapshot.
+            dataset_key: Dataset key.
+            items: List of DataSpecItem (with sourceItemIndex).
+        """
+        # Get snapshot indices in order
+        snapshot_indices = []
+        for run_index, item in enumerate(items):
+            snapshot_index = item.sourceItemIndex
+            if snapshot_index is None:
+                raise MaterializeError(
+                    f"Item at run index {run_index} missing sourceItemIndex"
+                )
+            snapshot_indices.append(snapshot_index)
+
+        src_path = snapshot.get_ndjson_path(dataset_key)
+        dst_path = data_dir / dataset_key / "object.ndjson"
+
+        if not src_path.exists():
+            raise MaterializeError(f"Source ndjson not found: {src_path}")
+
+        extract_ndjson_lines(src_path, dst_path, snapshot_indices)

fraclab_sdk/models/__init__.py

@@ -0,0 +1,52 @@
+"""Data models for SDK."""
+
+from fraclab_sdk.models.bundle_manifest import (
+    BundleManifest,
+    DatasetEntry,
+    DatasetEntryFile,
+    SpecFiles,
+)
+from fraclab_sdk.models.dataspec import DataSpec, DataSpecDataset, DataSpecItem
+from fraclab_sdk.models.drs import DRS, DRSDataset
+from fraclab_sdk.models.output_contract import (
+    BlobOutputSchema,
+    FrameOutputSchema,
+    ObjectOutputSchema,
+    OutputContract,
+    OutputDatasetContract,
+    OutputSchema,
+    ScalarOutputSchema,
+)
+from fraclab_sdk.models.run_output_manifest import (
+    ArtifactInfo,
+    OwnerRef,
+    RunInfo,
+    RunOutputDataset,
+    RunOutputItem,
+    RunOutputManifest,
+)
+
+__all__ = [
+    "BundleManifest",
+    "DatasetEntry",
+    "DatasetEntryFile",
+    "SpecFiles",
+    "DataSpec",
+    "DataSpecDataset",
+    "DataSpecItem",
+    "DRS",
+    "DRSDataset",
+    "OutputSchema",
+    "ScalarOutputSchema",
+    "FrameOutputSchema",
+    "ObjectOutputSchema",
+    "BlobOutputSchema",
+    "OutputContract",
+    "OutputDatasetContract",
+    "ArtifactInfo",
+    "OwnerRef",
+    "RunInfo",
+    "RunOutputDataset",
+    "RunOutputItem",
+    "RunOutputManifest",
+]

fraclab_sdk/models/bundle_manifest.py

@@ -0,0 +1,51 @@
+"""Bundle manifest model (Data Bundle Spec v1.0.0)."""
+
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict
+
+
+class SpecFiles(BaseModel):
+    """Specification files metadata."""
+
+    model_config = ConfigDict(extra="ignore")
+
+    dsPath: str = "ds.json"
+    drsPath: str = "drs.json"
+    dsSha256: str
+    drsSha256: str
+
+
+class DatasetEntryFile(BaseModel):
+    """Individual file entry in dataset files list."""
+
+    model_config = ConfigDict(extra="ignore")
+
+    path: str
+    sha256: str
+    bytes: int | None = None
+
+
+class DatasetEntry(BaseModel):
+    """Dataset entry in manifest."""
+
+    model_config = ConfigDict(extra="ignore")
+
+    layout: Literal["object_ndjson_lines", "frame_parquet_item_dirs"]
+    count: int
+    files: list[DatasetEntryFile] | None = None
+
+
+class BundleManifest(BaseModel):
+    """Manifest for a data bundle (Data Bundle Spec v1.0.0).
+
+    Contains metadata and integrity hashes for the bundle contents.
+    """
+
+    model_config = ConfigDict(extra="ignore")
+
+    bundleVersion: str
+    createdAtUs: int
+    specFiles: SpecFiles
+    dataRoot: str = "data"
+    datasets: dict[str, DatasetEntry]

fraclab_sdk/models/dataspec.py

@@ -0,0 +1,65 @@
+"""DataSpec model."""
+
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator
+
+
+class OwnerIds(BaseModel):
+    """Structured owner identifiers."""
+
+    model_config = ConfigDict(extra="ignore")
+
+    platformId: str | None = None
+    wellId: str | None = None
+    stageId: str | None = None
+
+
+class DataSpecItem(BaseModel):
+    """An item within a dataset."""
+
+    model_config = ConfigDict(extra="ignore")
+
+    owner: OwnerIds | str | None = None
+    resolutionParams: dict[str, Any] | None = None
+    range: dict[str, Any] | None = None
+    sourceItemIndex: int | None = None  # For traceability in run ds
+
+
+class DataSpecDataset(BaseModel):
+    """A dataset within a DataSpec."""
+
+    model_config = ConfigDict(extra="ignore", populate_by_name=True)
+
+    datasetKey: str = Field(alias="key")
+    resourceType: str | None = None
+    layout: str | None = None  # e.g., "frame_parquet_item_dirs" or "object_ndjson_lines"
+    items: list[DataSpecItem] = Field(default_factory=list)
+
+
+class DataSpec(BaseModel):
+    """Data specification describing datasets and their items."""
+
+    model_config = ConfigDict(extra="ignore", populate_by_name=True)
+
+    schemaVersion: str | None = None
+    datasets: list[DataSpecDataset] = Field(default_factory=list)
+
+    @field_validator("datasets", mode="before")
+    @classmethod
+    def _coerce_datasets(cls, v):
+        """Accept mapping form {'key': {...}} by converting to list."""
+        if isinstance(v, dict):
+            return [{"key": k, **(val or {})} for k, val in v.items()]
+        return v
+
+    def get_dataset(self, dataset_key: str) -> DataSpecDataset | None:
+        """Get a dataset by key."""
+        for ds in self.datasets:
+            if ds.datasetKey == dataset_key:
+                return ds
+        return None
+
+    def get_dataset_keys(self) -> list[str]:
+        """Get all dataset keys."""
+        return [ds.datasetKey for ds in self.datasets]