juniper-data 0.4.2__py3-none-any.whl

juniper_data/api/security.py

@@ -0,0 +1,238 @@
+"""API security: authentication and rate limiting middleware."""
+
+import time
+from collections import defaultdict
+from threading import Lock
+
+from fastapi import HTTPException, Request, status
+from fastapi.security import APIKeyHeader
+
+from .settings import get_settings
+
+api_key_header = APIKeyHeader(name="X-API-Key", auto_error=False)
+
+
+class APIKeyAuth:
+    """API key authentication handler.
+
+    Validates requests against configured API keys. When no API keys are
+    configured, authentication is disabled (open access mode for development).
+    """
+
+    def __init__(self, api_keys: list[str] | None = None) -> None:
+        """Initialize with optional list of valid API keys.
+
+        Args:
+            api_keys: List of valid API keys. If None or empty, auth is disabled.
+        """
+        self._api_keys: set[str] = set(api_keys) if api_keys else set()
+        self._enabled = len(self._api_keys) > 0
+
+    @property
+    def enabled(self) -> bool:
+        """Check if authentication is enabled."""
+        return self._enabled
+
+    def validate(self, api_key: str | None) -> bool:
+        """Validate an API key.
+
+        Args:
+            api_key: The API key to validate.
+
+        Returns:
+            True if auth is disabled or key is valid, False otherwise.
+        """
+        if not self._enabled:
+            return True
+        if api_key is None:
+            return False
+        return api_key in self._api_keys
+
+    async def __call__(self, request: Request) -> str | None:
+        """FastAPI dependency for API key validation.
+
+        Args:
+            request: The incoming request.
+
+        Returns:
+            The validated API key, or None if auth is disabled.
+
+        Raises:
+            HTTPException: 401 if auth is enabled and key is invalid/missing.
+        """
+        api_key = request.headers.get("X-API-Key")
+
+        if not self._enabled:
+            return None
+
+        if api_key is None:
+            raise HTTPException(
+                status_code=status.HTTP_401_UNAUTHORIZED,
+                detail="Missing API key. Provide X-API-Key header.",
+            )
+
+        if not self.validate(api_key):
+            raise HTTPException(
+                status_code=status.HTTP_401_UNAUTHORIZED,
+                detail="Invalid API key.",
+            )
+
+        return api_key
+
+
+class RateLimiter:
+    """In-memory fixed-window rate limiter.
+
+    Tracks request counts per key within fixed time windows. Thread-safe
+    implementation suitable for single-process deployments.
+    """
+
+    def __init__(
+        self,
+        requests_per_minute: int = 60,
+        window_seconds: int = 60,
+        enabled: bool = True,
+    ) -> None:
+        """Initialize the rate limiter.
+
+        Args:
+            requests_per_minute: Maximum requests allowed per window.
+            window_seconds: Window duration in seconds.
+            enabled: Whether rate limiting is enabled.
+        """
+        self._limit = requests_per_minute
+        self._window = window_seconds
+        self._enabled = enabled
+        self._counters: dict[str, tuple[int, float]] = defaultdict(lambda: (0, 0.0))
+        self._lock = Lock()
+
+    @property
+    def enabled(self) -> bool:
+        """Check if rate limiting is enabled."""
+        return self._enabled
+
+    @property
+    def limit(self) -> int:
+        """Get the rate limit."""
+        return self._limit
+
+    @property
+    def window(self) -> int:
+        """Get the window duration in seconds."""
+        return self._window
+
+    def _get_key(self, request: Request, api_key: str | None) -> str:
+        """Generate a rate limit key for the request.
+
+        Uses API key if available, otherwise falls back to client IP.
+
+        Args:
+            request: The incoming request.
+            api_key: The authenticated API key, if any.
+
+        Returns:
+            A string key for rate limiting.
+        """
+        if api_key:
+            return f"key:{api_key}"
+        client_ip = request.client.host if request.client else "unknown"
+        return f"ip:{client_ip}"
+
+    def check(self, key: str) -> tuple[bool, int, int]:
+        """Check if a request is allowed under rate limit.
+
+        Args:
+            key: The rate limit key.
+
+        Returns:
+            Tuple of (allowed, remaining, reset_seconds).
+        """
+        if not self._enabled:
+            return (True, self._limit, self._window)
+
+        now = time.time()
+
+        with self._lock:
+            count, window_start = self._counters[key]
+
+            if now - window_start >= self._window:
+                self._counters[key] = (1, now)
+                return (True, self._limit - 1, self._window)
+
+            if count >= self._limit:
+                reset_in = int(self._window - (now - window_start))
+                return (False, 0, reset_in)
+
+            self._counters[key] = (count + 1, window_start)
+            return (True, self._limit - count - 1, int(self._window - (now - window_start)))
+
+    async def __call__(self, request: Request, api_key: str | None = None) -> None:
+        """FastAPI dependency for rate limit checking.
+
+        Args:
+            request: The incoming request.
+            api_key: The authenticated API key, if any.
+
+        Raises:
+            HTTPException: 429 if rate limit exceeded.
+        """
+        if not self._enabled:
+            return
+
+        key = self._get_key(request, api_key)
+        allowed, remaining, reset_in = self.check(key)
+
+        request.state.rate_limit_remaining = remaining
+        request.state.rate_limit_reset = reset_in
+
+        if not allowed:
+            raise HTTPException(
+                status_code=status.HTTP_429_TOO_MANY_REQUESTS,
+                detail=f"Rate limit exceeded. Try again in {reset_in} seconds.",
+                headers={
+                    "X-RateLimit-Limit": str(self._limit),
+                    "X-RateLimit-Remaining": "0",
+                    "X-RateLimit-Reset": str(reset_in),
+                    "Retry-After": str(reset_in),
+                },
+            )
+
+    def reset(self) -> None:
+        """Reset all rate limit counters. Useful for testing."""
+        with self._lock:
+            self._counters.clear()
+
+
+_api_key_auth: APIKeyAuth | None = None
+_rate_limiter: RateLimiter | None = None
+
+
+def get_api_key_auth() -> APIKeyAuth:
+    """Get the global API key auth handler, creating if needed."""
+    global _api_key_auth
+    if _api_key_auth is None:
+        settings = get_settings()
+        api_keys = getattr(settings, "api_keys", None)
+        _api_key_auth = APIKeyAuth(api_keys)
+    return _api_key_auth
+
+
+def get_rate_limiter() -> RateLimiter:
+    """Get the global rate limiter, creating if needed."""
+    global _rate_limiter
+    if _rate_limiter is None:
+        settings = get_settings()
+        enabled = getattr(settings, "rate_limit_enabled", False)
+        requests_per_minute = getattr(settings, "rate_limit_requests_per_minute", 60)
+        _rate_limiter = RateLimiter(
+            requests_per_minute=requests_per_minute,
+            enabled=enabled,
+        )
+    return _rate_limiter
+
+
+def reset_security_state() -> None:
+    """Reset global security state. Useful for testing."""
+    global _api_key_auth, _rate_limiter
+    _api_key_auth = None
+    _rate_limiter = None

juniper_data/api/settings.py

@@ -0,0 +1,109 @@
+"""API configuration settings using pydantic-settings."""
+
+# import json
+from functools import lru_cache
+
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+# Define Safe and Reasonable Defaults for API Model Config
+_JUNIPER_DATA_ENV_PREFIX: str = "JUNIPER_DATA_"
+_JUNIPER_DATA_ENV_PREFIX_DEFAULT: str = _JUNIPER_DATA_ENV_PREFIX
+
+_JUNIPER_DATA_ENV_FILE: str = ".env"
+_JUNIPER_DATA_ENV_FILE_DEFAULT: str = _JUNIPER_DATA_ENV_FILE
+
+_JUNIPER_DATA_ENV_FILE_ENCODING: str = "utf-8"
+_JUNIPER_DATA_ENV_FILE_ENCODING_DEFAULT: str = _JUNIPER_DATA_ENV_FILE_ENCODING
+
+_JUNIPER_DATA_ENV_CASE_SENSITIVE_ENABLED: bool = True
+_JUNIPER_DATA_ENV_CASE_SENSITIVE_DISABLED: bool = False
+_JUNIPER_DATA_ENV_CASE_SENSITIVE_DEFAULT: bool = _JUNIPER_DATA_ENV_CASE_SENSITIVE_DISABLED
+
+_JUNIPER_DATA_ENV_EXTRA_DISABLED: str = "ignore"
+_JUNIPER_DATA_ENV_EXTRA_DEFAULT: str = _JUNIPER_DATA_ENV_EXTRA_DISABLED
+
+# Define Safe and Reasonable Defaults for API Settings
+_JUNIPER_DATA_API_DATASET_PATH: str = "./data/datasets"
+_JUNIPER_DATA_API_STORAGE_PATH_DEFAULT: str = _JUNIPER_DATA_API_DATASET_PATH
+
+_JUNIPER_DATA_API_HOST_GLOBAL: str = "0.0.0.0"  # nosec B104
+_JUNIPER_DATA_API_HOST_LOCAL: str = "127.0.0.1"
+_JUNIPER_DATA_API_HOST_DEFAULT: str = _JUNIPER_DATA_API_HOST_LOCAL
+
+_JUNIPER_DATA_API_PORT: int = 8100
+_JUNIPER_DATA_API_PORT_DEFAULT: int = _JUNIPER_DATA_API_PORT
+
+_JUNIPER_DATA_API_LOGLEVEL_TRACE: str = "TRACE"
+_JUNIPER_DATA_API_LOGLEVEL_VERBOSE: str = "VERBOSE"
+_JUNIPER_DATA_API_LOGLEVEL_DEBUG: str = "DEBUG"
+_JUNIPER_DATA_API_LOGLEVEL_INFO: str = "INFO"
+_JUNIPER_DATA_API_LOGLEVEL_WARNING: str = "WARNING"
+_JUNIPER_DATA_API_LOGLEVEL_ERROR: str = "ERROR"
+_JUNIPER_DATA_API_LOGLEVEL_CRITICAL: str = "CRITICAL"
+_JUNIPER_DATA_API_LOGLEVEL_FATAL: str = "FATAL"
+_JUNIPER_DATA_API_LOGLEVEL_DEFAULT: str = _JUNIPER_DATA_API_LOGLEVEL_INFO
+
+_JUNIPER_DATA_API_RATELIMIT_DISABLED: bool = False
+_JUNIPER_DATA_API_RATELIMIT_ENABLED: bool = True
+_JUNIPER_DATA_API_RATELIMIT_ACTIVE_DEFAULT: bool = _JUNIPER_DATA_API_RATELIMIT_DISABLED
+
+_JUNIPER_DATA_API_RATELIMIT_VALUE_SLOW: int = 30  # Requests per Minute
+_JUNIPER_DATA_API_RATELIMIT_VALUE_MID: int = 60  # Requests per Minute
+_JUNIPER_DATA_API_RATELIMIT_VALUE_FAST: int = 120  # Requests per Minute
+_JUNIPER_DATA_API_RATELIMIT_DEFAULT: int = _JUNIPER_DATA_API_RATELIMIT_VALUE_MID
+
+_JUNIPER_DATA_API_CORS_ORIGINS_ALL: list[str] = ["*"]
+_JUNIPER_DATA_API_CORS_ORIGINS_NONE: list[str] = []
+_JUNIPER_DATA_API_CORS_ORIGINS_DEFAULT: list[str] = _JUNIPER_DATA_API_CORS_ORIGINS_ALL
+
+
+_JUNIPER_DATA_API_KEYS_LIST_EMPTY: list[str] | None = None
+_JUNIPER_DATA_API_KEYS_LIST_VALUES: list[str] | None = []
+_JUNIPER_DATA_API_KEYS_LIST_DEFAULT: list[str] | None = _JUNIPER_DATA_API_KEYS_LIST_EMPTY
+
+
+class Settings(BaseSettings):
+    """Application settings loaded from environment variables.
+
+    All settings can be overridden via environment variables with the
+    JUNIPER_DATA_ prefix (e.g., JUNIPER_DATA_STORAGE_PATH).
+
+    Security Settings:
+        - api_keys: JSON list of comma-separated, valid API keys (e.g., ["key1,key2"] ).
+        - If empty, authentication is disabled (open access).
+        - rate_limit_enabled: Enable/disable rate limiting.
+        - rate_limit_requests_per_minute: Max requests per minute per client.
+    """
+
+    model_config = SettingsConfigDict(
+        env_prefix=_JUNIPER_DATA_ENV_PREFIX_DEFAULT,
+        env_file=_JUNIPER_DATA_ENV_FILE_DEFAULT,
+        env_file_encoding=_JUNIPER_DATA_ENV_FILE_ENCODING_DEFAULT,
+        case_sensitive=_JUNIPER_DATA_ENV_CASE_SENSITIVE_DEFAULT,
+        extra=_JUNIPER_DATA_ENV_EXTRA_DEFAULT,
+    )
+
+    # storage_path: str = "./data/datasets"
+    storage_path: str = _JUNIPER_DATA_API_STORAGE_PATH_DEFAULT
+
+    # Default to a more restrictive binding (e.g., 127.0.0.1) for general, non-containerized environments.
+    # To provide external access and allow listening on all interfaces,
+    #   for compatibility with containerized deployments (e.g., Docker, Kubernetes),
+    #   override JUNIPER_DATA_HOST (e.g., to 0.0.0.0).
+    # Note: When setting JUNIPER_DATA_HOST to 0.0.0.0, use firewall/security groups or reverse proxies to control access.
+    host: str = _JUNIPER_DATA_API_HOST_DEFAULT
+    port: int = _JUNIPER_DATA_API_PORT_DEFAULT
+    log_level: str = _JUNIPER_DATA_API_LOGLEVEL_DEFAULT
+    cors_origins: list[str] = _JUNIPER_DATA_API_CORS_ORIGINS_DEFAULT
+
+    # api_keys: list[str] | None = _JUNIPER_DATA_API_KEYS_LIST_DEFAULT
+    # api_keys: JSON[list[str]] | None = _JUNIPER_DATA_API_KEYS_LIST_DEFAULT
+    api_keys: list[str] | None = _JUNIPER_DATA_API_KEYS_LIST_DEFAULT
+    rate_limit_enabled: bool = _JUNIPER_DATA_API_RATELIMIT_ACTIVE_DEFAULT
+    rate_limit_requests_per_minute: int = _JUNIPER_DATA_API_RATELIMIT_DEFAULT
+
+
+@lru_cache
+def get_settings() -> Settings:
+    """Get cached application settings."""
+    return Settings()

juniper_data/core/__init__.py

@@ -0,0 +1,32 @@
+"""Core module for Juniper Data."""
+
+from juniper_data.core.artifacts import arrays_to_bytes, compute_checksum, load_npz, save_npz
+from juniper_data.core.dataset_id import generate_dataset_id
+from juniper_data.core.models import (
+    CreateDatasetRequest,
+    CreateDatasetResponse,
+    DatasetMeta,
+    GeneratorInfo,
+    PreviewData,
+)
+from juniper_data.core.split import shuffle_and_split, shuffle_data, split_data
+
+__all__ = [
+    # Dataset ID
+    "generate_dataset_id",
+    # Split utilities
+    "shuffle_and_split",
+    "shuffle_data",
+    "split_data",
+    # Models
+    "CreateDatasetRequest",
+    "CreateDatasetResponse",
+    "DatasetMeta",
+    "GeneratorInfo",
+    "PreviewData",
+    # Artifacts
+    "arrays_to_bytes",
+    "compute_checksum",
+    "load_npz",
+    "save_npz",
+]

juniper_data/core/artifacts.py

@@ -0,0 +1,63 @@
+"""Artifact utilities for NPZ file handling and checksum computation."""
+
+import hashlib
+import io
+from pathlib import Path
+
+import numpy as np
+
+
+def save_npz(path: Path, arrays: dict[str, np.ndarray]) -> None:
+    """Save arrays to NPZ file.
+
+    Args:
+        path: Path to save the NPZ file.
+        arrays: Dictionary mapping array names to numpy arrays.
+    """
+    np.savez(path, **arrays)  # type: ignore[arg-type]  # numpy stubs incomplete for **kwargs
+
+
+def load_npz(path: Path) -> dict[str, np.ndarray]:
+    """Load arrays from NPZ file.
+
+    Args:
+        path: Path to the NPZ file.
+
+    Returns:
+        Dictionary mapping array names to numpy arrays.
+    """
+    with np.load(path) as data:
+        return {key: data[key] for key in data.files}
+
+
+def arrays_to_bytes(arrays: dict[str, np.ndarray]) -> bytes:
+    """Convert arrays to NPZ bytes for streaming response.
+
+    Args:
+        arrays: Dictionary mapping array names to numpy arrays.
+
+    Returns:
+        Bytes representation of the NPZ file.
+    """
+    buffer = io.BytesIO()
+    # Ensure a stable serialization order by sorting keys before saving.
+    ordered_arrays = {key: arrays[key] for key in sorted(arrays.keys())}
+    np.savez(buffer, **ordered_arrays)  # type: ignore[arg-type]  # numpy stubs incomplete for **kwargs
+    buffer.seek(0)
+    return buffer.read()
+
+
+def compute_checksum(arrays: dict[str, np.ndarray]) -> str:
+    """Compute SHA-256 checksum of arrays for integrity verification.
+
+    The checksum is computed over the NPZ byte representation of the arrays,
+    ensuring consistent results across different systems.
+
+    Args:
+        arrays: Dictionary mapping array names to numpy arrays.
+
+    Returns:
+        SHA-256 hex digest of the arrays.
+    """
+    data = arrays_to_bytes(arrays)
+    return hashlib.sha256(data).hexdigest()

juniper_data/core/dataset_id.py

@@ -0,0 +1,38 @@
+"""Dataset ID generation utilities.
+
+This module provides deterministic ID generation for datasets based on
+generator name, version, and parameters.
+"""
+
+import hashlib
+import json
+from typing import Any
+
+
+def generate_dataset_id(generator: str, version: str, params: dict[str, Any]) -> str:
+    """Generate a deterministic hash-based ID from generator metadata and params.
+
+    Creates a unique, reproducible identifier for a dataset configuration by
+    hashing the canonical JSON representation of the generator name, version,
+    and parameters.
+
+    Args:
+        generator: Name of the generator (e.g., "spiral").
+        version: Version string (e.g., "v1.0.0").
+        params: Dictionary of generator parameters.
+
+    Returns:
+        Dataset ID in format "{generator}-{version}-{hash[:16]}".
+        Example: "spiral-v1.0.0-a3f8e12b4c567890"
+    """
+    canonical_data = {
+        "generator": generator,
+        "version": version,
+        "params": params,
+    }
+
+    canonical_json = json.dumps(canonical_data, sort_keys=True, separators=(",", ":"))
+
+    hash_digest = hashlib.sha256(canonical_json.encode("utf-8")).hexdigest()
+
+    return f"{generator}-{version}-{hash_digest[:16]}"

juniper_data/core/models.py

@@ -0,0 +1,135 @@
+"""Core Pydantic models for dataset metadata and API responses."""
+
+from datetime import datetime
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+
+class DatasetMeta(BaseModel):
+    """Dataset metadata (always small, JSON-safe)."""
+
+    # Identity
+    dataset_id: str
+    generator: str
+    generator_version: str
+
+    # Generation Parameters
+    params: dict[str, Any]
+
+    # Shape Information
+    n_samples: int
+    n_features: int
+    n_classes: int
+    n_train: int
+    n_test: int
+
+    # Class Distribution (str keys for JSON compatibility)
+    class_distribution: dict[str, int]
+
+    # Artifacts
+    artifact_formats: list[str] = Field(default_factory=lambda: ["npz"])
+
+    # Timestamps
+    created_at: datetime
+
+    # Optional fields
+    checksum: str | None = None
+
+    # Lifecycle management (DATA-016)
+    tags: list[str] = Field(default_factory=list)
+    ttl_seconds: int | None = None
+    expires_at: datetime | None = None
+    last_accessed_at: datetime | None = None
+    access_count: int = 0
+
+
+class CreateDatasetRequest(BaseModel):
+    """Request model for creating a new dataset."""
+
+    generator: str
+    params: dict[str, Any] = Field(default_factory=dict)
+    persist: bool = True
+    tags: list[str] = Field(default_factory=list)
+    ttl_seconds: int | None = Field(default=None, ge=1, description="Time-to-live in seconds")
+
+
+class CreateDatasetResponse(BaseModel):
+    """Response model for dataset creation."""
+
+    dataset_id: str
+    generator: str
+    meta: DatasetMeta
+    artifact_url: str
+
+
+class GeneratorInfo(BaseModel):
+    """Information about an available generator."""
+
+    name: str
+    version: str
+    description: str
+    params_schema: dict[str, Any] = Field(alias="schema")  # JSON schema for params
+
+
+class PreviewData(BaseModel):
+    """Preview subset of a dataset for visualization."""
+
+    n_samples: int
+    X_sample: list[list[float]]
+    y_sample: list[list[float]]
+
+
+class DatasetListFilter(BaseModel):
+    """Filter criteria for listing datasets."""
+
+    generator: str | None = None
+    tags: list[str] | None = None
+    tags_match: str = Field(default="any", pattern="^(any|all)$")
+    created_after: datetime | None = None
+    created_before: datetime | None = None
+    min_samples: int | None = Field(default=None, ge=1)
+    max_samples: int | None = Field(default=None, ge=1)
+    include_expired: bool = False
+
+
+class DatasetListResponse(BaseModel):
+    """Response model for filtered dataset listing."""
+
+    datasets: list[DatasetMeta]
+    total: int
+    limit: int
+    offset: int
+
+
+class BatchDeleteRequest(BaseModel):
+    """Request model for batch delete operation."""
+
+    dataset_ids: list[str] = Field(min_length=1, max_length=100)
+
+
+class BatchDeleteResponse(BaseModel):
+    """Response model for batch delete operation."""
+
+    deleted: list[str]
+    not_found: list[str]
+    total_deleted: int
+
+
+class UpdateTagsRequest(BaseModel):
+    """Request model for updating dataset tags."""
+
+    add_tags: list[str] = Field(default_factory=list)
+    remove_tags: list[str] = Field(default_factory=list)
+
+
+class DatasetStats(BaseModel):
+    """Aggregate statistics about stored datasets."""
+
+    total_datasets: int
+    total_samples: int
+    by_generator: dict[str, int]
+    by_tag: dict[str, int]
+    oldest_created_at: datetime | None = None
+    newest_created_at: datetime | None = None
+    expired_count: int = 0