dataenginex 0.3.4__py3-none-any.whl

dataenginex/data/registry.py

@@ -0,0 +1,148 @@
+"""
+Schema registry — versioned schema management for DEX datasets.
+
+Stores schema definitions (as JSON-serialisable dicts) with
+semantic versioning, allowing pipelines to validate data against
+a specific schema revision and to track schema evolution.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from loguru import logger
+
+
+@dataclass
+class SchemaVersion:
+    """An immutable snapshot of a schema at a particular version."""
+
+    name: str
+    version: str  # semver string, e.g. "1.2.0"
+    fields: dict[str, str]  # field_name → type_description
+    required_fields: list[str] = field(default_factory=list)
+    description: str = ""
+    created_at: datetime = field(default_factory=lambda: datetime.now(tz=UTC))
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "name": self.name,
+            "version": self.version,
+            "fields": self.fields,
+            "required_fields": self.required_fields,
+            "description": self.description,
+            "created_at": self.created_at.isoformat(),
+            "metadata": self.metadata,
+        }
+
+    def validate_record(self, record: dict[str, Any]) -> tuple[bool, list[str]]:
+        """Check that *record* has all required fields.
+
+        Returns ``(is_valid, errors)`` where *errors* lists the missing
+        required fields.
+        """
+        missing = [f for f in self.required_fields if f not in record]
+        return len(missing) == 0, [f"Missing required field: {f}" for f in missing]
+
+
+class SchemaRegistry:
+    """In-process schema registry backed by an optional JSON file.
+
+    Parameters
+    ----------
+    persist_path:
+        If given, schemas are saved/loaded from this JSON file so they
+        survive across process restarts.
+    """
+
+    def __init__(self, persist_path: str | Path | None = None) -> None:
+        # schema_name → [SchemaVersion …] (ordered oldest → newest)
+        self._schemas: dict[str, list[SchemaVersion]] = {}
+        self._persist_path = Path(persist_path) if persist_path else None
+        if self._persist_path and self._persist_path.exists():
+            self._load()
+
+    # -- public API ----------------------------------------------------------
+
+    def register(self, schema: SchemaVersion) -> SchemaVersion:
+        """Register a new schema version.  Duplicate versions are rejected."""
+        versions = self._schemas.setdefault(schema.name, [])
+        existing = {v.version for v in versions}
+        if schema.version in existing:
+            raise ValueError(
+                f"Schema {schema.name!r} version {schema.version} already registered"
+            )
+        versions.append(schema)
+        logger.info("Registered schema %s v%s", schema.name, schema.version)
+        self._save()
+        return schema
+
+    def get_latest(self, name: str) -> SchemaVersion | None:
+        """Return the most recently registered version for *name*."""
+        versions = self._schemas.get(name)
+        if not versions:
+            return None
+        return versions[-1]
+
+    def get_version(self, name: str, version: str) -> SchemaVersion | None:
+        """Return a specific version, or *None* if not found."""
+        for v in self._schemas.get(name, []):
+            if v.version == version:
+                return v
+        return None
+
+    def list_schemas(self) -> list[str]:
+        """Return all registered schema names."""
+        return list(self._schemas.keys())
+
+    def list_versions(self, name: str) -> list[str]:
+        """Return all registered versions for *name* (oldest first)."""
+        return [v.version for v in self._schemas.get(name, [])]
+
+    def validate(
+        self, name: str, record: dict[str, Any], version: str | None = None
+    ) -> tuple[bool, list[str]]:
+        """Validate *record* against a schema.
+
+        If *version* is ``None`` the latest version is used.
+        """
+        schema = (
+            self.get_version(name, version) if version else self.get_latest(name)
+        )
+        if schema is None:
+            return False, [f"Schema {name!r} (version={version}) not found"]
+        return schema.validate_record(record)
+
+    # -- persistence ---------------------------------------------------------
+
+    def _save(self) -> None:
+        if not self._persist_path:
+            return
+        data: dict[str, list[dict[str, Any]]] = {}
+        for name, versions in self._schemas.items():
+            data[name] = [v.to_dict() for v in versions]
+        self._persist_path.parent.mkdir(parents=True, exist_ok=True)
+        self._persist_path.write_text(json.dumps(data, indent=2, default=str))
+
+    def _load(self) -> None:
+        if not self._persist_path or not self._persist_path.exists():
+            return
+        raw = json.loads(self._persist_path.read_text())
+        for name, versions in raw.items():
+            self._schemas[name] = [
+                SchemaVersion(
+                    name=v["name"],
+                    version=v["version"],
+                    fields=v["fields"],
+                    required_fields=v.get("required_fields", []),
+                    description=v.get("description", ""),
+                    metadata=v.get("metadata", {}),
+                )
+                for v in versions
+            ]
+        logger.info("Loaded %d schemas from %s", len(self._schemas), self._persist_path)

dataenginex/lakehouse/__init__.py

@@ -0,0 +1,22 @@
+"""
+dex-lakehouse — Storage backends, data catalog, and partitioning (Epic #39).
+
+Provides:
+    - ``ParquetStorage`` / ``JsonStorage`` — concrete ``StorageBackend`` impls
+    - ``DataCatalog`` — registry of datasets with metadata
+    - ``PartitionStrategy`` — time/hash/range-based partitioning helpers
+"""
+
+from .catalog import CatalogEntry, DataCatalog
+from .partitioning import DatePartitioner, HashPartitioner, PartitionStrategy
+from .storage import JsonStorage, ParquetStorage
+
+__all__ = [
+    "CatalogEntry",
+    "DataCatalog",
+    "DatePartitioner",
+    "HashPartitioner",
+    "JsonStorage",
+    "ParquetStorage",
+    "PartitionStrategy",
+]

dataenginex/lakehouse/catalog.py

@@ -0,0 +1,145 @@
+"""
+Data catalog — registry of lakehouse datasets with metadata.
+
+``DataCatalog`` keeps track of every dataset written to the lakehouse,
+recording its layer, format, location, schema snapshot, and record counts
+so that downstream consumers can discover available data.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import asdict, dataclass, field
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from loguru import logger
+
+
+@dataclass
+class CatalogEntry:
+    """Metadata about a single dataset in the lakehouse."""
+
+    name: str
+    layer: str  # "bronze", "silver", "gold"
+    format: str  # "parquet", "json", "delta"
+    location: str  # file path or table ref
+    record_count: int = 0
+    schema_fields: list[str] = field(default_factory=list)
+    description: str = ""
+    owner: str = ""
+    tags: list[str] = field(default_factory=list)
+    created_at: datetime = field(default_factory=lambda: datetime.now(tz=UTC))
+    updated_at: datetime = field(default_factory=lambda: datetime.now(tz=UTC))
+    metadata: dict[str, Any] = field(default_factory=dict)
+    version: int = 1
+
+    def to_dict(self) -> dict[str, Any]:
+        d = asdict(self)
+        d["created_at"] = self.created_at.isoformat()
+        d["updated_at"] = self.updated_at.isoformat()
+        return d
+
+
+class DataCatalog:
+    """In-process data catalog backed by an optional JSON file.
+
+    Parameters
+    ----------
+    persist_path:
+        When set, catalog entries are persisted to this JSON file.
+    """
+
+    def __init__(self, persist_path: str | Path | None = None) -> None:
+        self._entries: dict[str, CatalogEntry] = {}
+        self._persist_path = Path(persist_path) if persist_path else None
+        if self._persist_path and self._persist_path.exists():
+            self._load()
+
+    # -- public API ----------------------------------------------------------
+
+    def register(self, entry: CatalogEntry) -> CatalogEntry:
+        """Register or update a dataset entry."""
+        existing = self._entries.get(entry.name)
+        if existing:
+            entry.version = existing.version + 1
+            entry.created_at = existing.created_at
+        entry.updated_at = datetime.now(tz=UTC)
+        self._entries[entry.name] = entry
+        logger.info(
+            "Catalog registered: %s (layer=%s, v%d)",
+            entry.name, entry.layer, entry.version,
+        )
+        self._save()
+        return entry
+
+    def get(self, name: str) -> CatalogEntry | None:
+        """Retrieve an entry by name."""
+        return self._entries.get(name)
+
+    def search(
+        self,
+        *,
+        layer: str | None = None,
+        tags: list[str] | None = None,
+        owner: str | None = None,
+        name_contains: str | None = None,
+    ) -> list[CatalogEntry]:
+        """Search entries by criteria."""
+        results = list(self._entries.values())
+        if layer:
+            results = [e for e in results if e.layer == layer]
+        if tags:
+            tag_set = set(tags)
+            results = [e for e in results if tag_set.issubset(set(e.tags))]
+        if owner:
+            results = [e for e in results if e.owner == owner]
+        if name_contains:
+            results = [e for e in results if name_contains.lower() in e.name.lower()]
+        return results
+
+    def list_all(self) -> list[CatalogEntry]:
+        """Return all catalog entries."""
+        return list(self._entries.values())
+
+    def delete(self, name: str) -> bool:
+        """Remove an entry by name."""
+        if name in self._entries:
+            del self._entries[name]
+            self._save()
+            return True
+        return False
+
+    def summary(self) -> dict[str, Any]:
+        """High-level catalog statistics."""
+        layers: dict[str, int] = {}
+        formats: dict[str, int] = {}
+        for e in self._entries.values():
+            layers[e.layer] = layers.get(e.layer, 0) + 1
+            formats[e.format] = formats.get(e.format, 0) + 1
+        return {
+            "total_datasets": len(self._entries),
+            "by_layer": layers,
+            "by_format": formats,
+        }
+
+    # -- persistence ---------------------------------------------------------
+
+    def _save(self) -> None:
+        if not self._persist_path:
+            return
+        self._persist_path.parent.mkdir(parents=True, exist_ok=True)
+        data = [e.to_dict() for e in self._entries.values()]
+        self._persist_path.write_text(json.dumps(data, indent=2, default=str))
+
+    def _load(self) -> None:
+        if not self._persist_path or not self._persist_path.exists():
+            return
+        raw = json.loads(self._persist_path.read_text())
+        for item in raw:
+            item.pop("created_at", None)
+            item.pop("updated_at", None)
+            entry = CatalogEntry(**item)
+            self._entries[entry.name] = entry
+        logger.info("Loaded %d catalog entries from %s", len(self._entries), self._persist_path)

dataenginex/lakehouse/partitioning.py

@@ -0,0 +1,99 @@
+"""
+Partitioning strategies for the DEX lakehouse.
+
+``PartitionStrategy`` is an ABC whose subclasses generate path segments
+used by storage backends to organise data into predictable directory trees.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from abc import ABC, abstractmethod
+from datetime import UTC, datetime
+from typing import Any
+
+
+class PartitionStrategy(ABC):
+    """Base class for partitioning strategies."""
+
+    @abstractmethod
+    def partition_key(self, record: dict[str, Any]) -> str:
+        """Return the partition path segment for *record*."""
+        ...
+
+    @abstractmethod
+    def partition_path(self, record: dict[str, Any], base: str = "") -> str:
+        """Return the full relative path (base + partition) for *record*."""
+        ...
+
+
+class DatePartitioner(PartitionStrategy):
+    """Partition by a date field using ``year=…/month=…/day=…`` layout.
+
+    Parameters
+    ----------
+    date_field:
+        Name of the record field containing a date/datetime value.
+    granularity:
+        ``"day"`` (default), ``"month"``, or ``"year"``.
+    """
+
+    def __init__(self, date_field: str = "created_at", granularity: str = "day") -> None:
+        self.date_field = date_field
+        if granularity not in ("day", "month", "year"):
+            raise ValueError(f"granularity must be day/month/year, got {granularity!r}")
+        self.granularity = granularity
+
+    def partition_key(self, record: dict[str, Any]) -> str:
+        dt = self._extract_date(record)
+        parts = [f"year={dt.year}"]
+        if self.granularity in ("month", "day"):
+            parts.append(f"month={dt.month:02d}")
+        if self.granularity == "day":
+            parts.append(f"day={dt.day:02d}")
+        return "/".join(parts)
+
+    def partition_path(self, record: dict[str, Any], base: str = "") -> str:
+        key = self.partition_key(record)
+        return f"{base}/{key}" if base else key
+
+    def _extract_date(self, record: dict[str, Any]) -> datetime:
+        value = record.get(self.date_field)
+        if isinstance(value, datetime):
+            return value
+        if isinstance(value, str):
+            # Try ISO format
+            try:
+                return datetime.fromisoformat(value)
+            except ValueError:
+                pass
+        # Fallback to now
+        return datetime.now(tz=UTC)
+
+
+class HashPartitioner(PartitionStrategy):
+    """Partition by a hash of one or more fields, distributing across *n_buckets*.
+
+    Parameters
+    ----------
+    fields:
+        Record fields whose values are hashed.
+    n_buckets:
+        Number of hash buckets (directories).
+    """
+
+    def __init__(self, fields: list[str], n_buckets: int = 16) -> None:
+        if not fields:
+            raise ValueError("At least one field is required for hash partitioning")
+        self.fields = fields
+        self.n_buckets = max(1, n_buckets)
+
+    def partition_key(self, record: dict[str, Any]) -> str:
+        content = "|".join(str(record.get(f, "")) for f in self.fields)
+        digest = hashlib.md5(content.encode()).hexdigest()  # noqa: S324
+        bucket = int(digest, 16) % self.n_buckets
+        return f"bucket={bucket:04d}"
+
+    def partition_path(self, record: dict[str, Any], base: str = "") -> str:
+        key = self.partition_key(record)
+        return f"{base}/{key}" if base else key

dataenginex/lakehouse/storage.py

@@ -0,0 +1,177 @@
+"""
+Concrete storage backends for the DEX lakehouse.
+
+Both ``ParquetStorage`` and ``JsonStorage`` implement the
+``StorageBackend`` ABC from ``dataenginex.core.medallion_architecture`` so
+they can be used interchangeably by the ``DualStorage`` layer.
+
+``ParquetStorage`` delegates to *pyarrow* when available; otherwise it
+falls back to ``JsonStorage`` with a logged warning.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from loguru import logger
+
+from dataenginex.core.medallion_architecture import StorageBackend, StorageFormat
+
+# Try importing pyarrow — optional heavyweight dependency
+try:
+    import pyarrow as pa  # type: ignore[import-not-found]
+    import pyarrow.parquet as pq  # type: ignore[import-not-found]
+
+    _HAS_PYARROW = True
+except ImportError:
+    _HAS_PYARROW = False
+
+
+# ---------------------------------------------------------------------------
+# JSON storage (always available)
+# ---------------------------------------------------------------------------
+
+class JsonStorage(StorageBackend):
+    """Simple JSON-file storage for development and testing.
+
+    Each ``write`` call serialises *data* (list of dicts) as a JSON array.
+    """
+
+    def __init__(self, base_path: str = "data") -> None:
+        self.base_path = Path(base_path)
+        self.base_path.mkdir(parents=True, exist_ok=True)
+        logger.info("JsonStorage initialised at %s", self.base_path)
+
+    def write(
+        self,
+        data: Any,
+        path: str,
+        format: StorageFormat = StorageFormat.PARQUET,
+    ) -> bool:
+        try:
+            full = self.base_path / f"{path}.json"
+            full.parent.mkdir(parents=True, exist_ok=True)
+            records = self._normalise(data)
+            full.write_text(json.dumps(records, indent=2, default=str))
+            logger.info("Wrote %d records to %s", len(records), full)
+            return True
+        except Exception as exc:
+            logger.error("JsonStorage write failed: %s", exc)
+            return False
+
+    def read(self, path: str, format: StorageFormat = StorageFormat.PARQUET) -> Any:
+        try:
+            full = self.base_path / f"{path}.json"
+            if not full.exists():
+                logger.warning("File not found: %s", full)
+                return None
+            return json.loads(full.read_text())
+        except Exception as exc:
+            logger.error("JsonStorage read failed: %s", exc)
+            return None
+
+    def delete(self, path: str) -> bool:
+        try:
+            full = self.base_path / f"{path}.json"
+            if full.exists():
+                full.unlink()
+                logger.info("Deleted %s", full)
+            return True
+        except Exception as exc:
+            logger.error("JsonStorage delete failed: %s", exc)
+            return False
+
+    @staticmethod
+    def _normalise(data: Any) -> list[dict[str, Any]]:
+        if isinstance(data, list):
+            return data
+        if isinstance(data, dict):
+            return [data]
+        return [{"value": data}]
+
+
+# ---------------------------------------------------------------------------
+# Parquet storage (requires pyarrow)
+# ---------------------------------------------------------------------------
+
+class ParquetStorage(StorageBackend):
+    """Parquet file storage backed by *pyarrow*.
+
+    Falls back to ``JsonStorage`` when *pyarrow* is not installed.
+    """
+
+    def __init__(self, base_path: str = "data", compression: str = "snappy") -> None:
+        self.base_path = Path(base_path)
+        self.base_path.mkdir(parents=True, exist_ok=True)
+        self.compression = compression
+
+        if _HAS_PYARROW:
+            logger.info("ParquetStorage initialised at %s (pyarrow available)", self.base_path)
+        else:
+            logger.warning(
+                "pyarrow not installed — ParquetStorage will use JSON fallback"
+            )
+            self._fallback = JsonStorage(str(self.base_path))
+
+    def write(
+        self,
+        data: Any,
+        path: str,
+        format: StorageFormat = StorageFormat.PARQUET,
+    ) -> bool:
+        if not _HAS_PYARROW:
+            return self._fallback.write(data, path, format)
+
+        try:
+            full = self.base_path / f"{path}.parquet"
+            full.parent.mkdir(parents=True, exist_ok=True)
+            records = self._to_records(data)
+            if not records:
+                logger.warning("No records to write to %s", full)
+                return False
+            table = pa.Table.from_pylist(records)
+            pq.write_table(table, str(full), compression=self.compression)
+            logger.info("Wrote %d records to %s", len(records), full)
+            return True
+        except Exception as exc:
+            logger.error("ParquetStorage write failed: %s", exc)
+            return False
+
+    def read(self, path: str, format: StorageFormat = StorageFormat.PARQUET) -> Any:
+        if not _HAS_PYARROW:
+            return self._fallback.read(path, format)
+
+        try:
+            full = self.base_path / f"{path}.parquet"
+            if not full.exists():
+                logger.warning("Parquet file not found: %s", full)
+                return None
+            table = pq.read_table(str(full))
+            return table.to_pylist()
+        except Exception as exc:
+            logger.error("ParquetStorage read failed: %s", exc)
+            return None
+
+    def delete(self, path: str) -> bool:
+        if not _HAS_PYARROW:
+            return self._fallback.delete(path)
+
+        try:
+            full = self.base_path / f"{path}.parquet"
+            if full.exists():
+                full.unlink()
+                logger.info("Deleted %s", full)
+            return True
+        except Exception as exc:
+            logger.error("ParquetStorage delete failed: %s", exc)
+            return False
+
+    @staticmethod
+    def _to_records(data: Any) -> list[dict[str, Any]]:
+        if isinstance(data, list):
+            return data
+        if isinstance(data, dict):
+            return [data]
+        return []

dataenginex/middleware/__init__.py

@@ -0,0 +1,19 @@
+"""
+Middleware - logging, metrics, tracing, and request handling.
+"""
+
+from .logging_config import APP_VERSION, configure_logging  # noqa: F401
+from .metrics import get_metrics  # noqa: F401
+from .metrics_middleware import PrometheusMetricsMiddleware  # noqa: F401
+from .request_logging import RequestLoggingMiddleware  # noqa: F401
+from .tracing import configure_tracing, instrument_fastapi  # noqa: F401
+
+__all__ = [
+    "configure_logging",
+    "APP_VERSION",
+    "get_metrics",
+    "PrometheusMetricsMiddleware",
+    "RequestLoggingMiddleware",
+    "configure_tracing",
+    "instrument_fastapi",
+]