dataenginex 0.3.4__py3-none-any.whl

dataenginex/warehouse/transforms.py

@@ -0,0 +1,206 @@
+"""
+Transform framework — declarative, composable data transformations.
+
+Each ``Transform`` is a named, reusable function that maps a record dict to a
+new record dict.  ``TransformPipeline`` chains multiple transforms and collects
+per-step metrics for observability.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import time
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from typing import Any
+
+from loguru import logger
+
+
+@dataclass
+class TransformResult:
+    """Outcome of running a pipeline over a batch of records."""
+
+    input_count: int
+    output_count: int
+    dropped_count: int = 0
+    error_count: int = 0
+    duration_ms: float = 0.0
+    records: list[dict[str, Any]] = field(default_factory=list)
+    step_metrics: list[dict[str, Any]] = field(default_factory=list)
+    completed_at: datetime = field(default_factory=lambda: datetime.now(tz=UTC))
+
+    @property
+    def success_rate(self) -> float:
+        return self.output_count / self.input_count if self.input_count else 0.0
+
+
+# ---------------------------------------------------------------------------
+# Single transform
+# ---------------------------------------------------------------------------
+
+class Transform(ABC):
+    """Base class for a single data transform step.
+
+    Subclass and implement ``apply`` which receives one record and returns
+    either a transformed record or *None* to drop it.
+    """
+
+    def __init__(self, name: str, description: str = "") -> None:
+        self.name = name
+        self.description = description
+
+    @abstractmethod
+    def apply(self, record: dict[str, Any]) -> dict[str, Any] | None:
+        """Transform *record* in place or return a new dict.
+
+        Return *None* to drop the record from the pipeline.
+        """
+        ...
+
+
+# ---------------------------------------------------------------------------
+# Built-in transforms
+# ---------------------------------------------------------------------------
+
+class RenameFieldsTransform(Transform):
+    """Rename keys in a record according to a mapping."""
+
+    def __init__(self, mapping: dict[str, str]) -> None:
+        super().__init__(name="rename_fields", description="Rename record fields")
+        self.mapping = mapping
+
+    def apply(self, record: dict[str, Any]) -> dict[str, Any]:
+        out = dict(record)
+        for old_key, new_key in self.mapping.items():
+            if old_key in out:
+                out[new_key] = out.pop(old_key)
+        return out
+
+
+class DropNullsTransform(Transform):
+    """Drop records that have *None* in any of the specified fields."""
+
+    def __init__(self, required_fields: list[str]) -> None:
+        super().__init__(name="drop_nulls", description="Drop records with null required fields")
+        self.required_fields = required_fields
+
+    def apply(self, record: dict[str, Any]) -> dict[str, Any] | None:
+        for f in self.required_fields:
+            if record.get(f) is None:
+                return None
+        return record
+
+
+class CastTypesTransform(Transform):
+    """Cast fields to target types (``int``, ``float``, ``str``, ``bool``)."""
+
+    _CASTERS: dict[str, type] = {"int": int, "float": float, "str": str, "bool": bool}
+
+    def __init__(self, type_map: dict[str, str]) -> None:
+        super().__init__(name="cast_types", description="Cast field types")
+        self.type_map = type_map
+
+    def apply(self, record: dict[str, Any]) -> dict[str, Any]:
+        out = dict(record)
+        for field_name, target in self.type_map.items():
+            if field_name in out and out[field_name] is not None:
+                caster = self._CASTERS.get(target)
+                if caster:
+                    with contextlib.suppress(ValueError, TypeError):
+                        out[field_name] = caster(out[field_name])
+        return out
+
+
+class AddTimestampTransform(Transform):
+    """Add a processing timestamp field to every record."""
+
+    def __init__(self, field_name: str = "processed_at") -> None:
+        super().__init__(name="add_timestamp", description="Add processing timestamp")
+        self.field_name = field_name
+
+    def apply(self, record: dict[str, Any]) -> dict[str, Any]:
+        out = dict(record)
+        out[self.field_name] = datetime.now(tz=UTC).isoformat()
+        return out
+
+
+class FilterTransform(Transform):
+    """Keep only records that match a predicate expression.
+
+    ``predicate`` receives a record dict and returns ``True`` to keep it.
+    """
+
+    def __init__(self, name: str, predicate: Any) -> None:
+        super().__init__(name=name, description="Filter records by predicate")
+        self._predicate = predicate
+
+    def apply(self, record: dict[str, Any]) -> dict[str, Any] | None:
+        if self._predicate(record):
+            return record
+        return None
+
+
+# ---------------------------------------------------------------------------
+# Pipeline
+# ---------------------------------------------------------------------------
+
+class TransformPipeline:
+    """Execute an ordered chain of ``Transform`` steps over a batch.
+
+    Example::
+
+        pipeline = TransformPipeline("bronze_to_silver")
+        pipeline.add(DropNullsTransform(["job_id", "company_name"]))
+        pipeline.add(CastTypesTransform({"salary_min": "float"}))
+        result = pipeline.run(records)
+    """
+
+    def __init__(self, name: str) -> None:
+        self.name = name
+        self._steps: list[Transform] = []
+
+    def add(self, transform: Transform) -> TransformPipeline:
+        self._steps.append(transform)
+        return self  # fluent API
+
+    def run(self, records: list[dict[str, Any]]) -> TransformResult:
+        start = time.perf_counter()
+        current = list(records)
+        step_metrics: list[dict[str, Any]] = []
+
+        for step in self._steps:
+            step_start = time.perf_counter()
+            before_count = len(current)
+            output: list[dict[str, Any]] = []
+
+            for rec in current:
+                try:
+                    result = step.apply(rec)
+                    if result is not None:
+                        output.append(result)
+                except Exception as exc:
+                    logger.warning(
+                        "Transform %s failed on record: %s", step.name, exc,
+                    )
+
+            step_duration = (time.perf_counter() - step_start) * 1000
+            step_metrics.append({
+                "step": step.name,
+                "input_count": before_count,
+                "output_count": len(output),
+                "dropped": before_count - len(output),
+                "duration_ms": round(step_duration, 2),
+            })
+            current = output
+
+        total_duration = (time.perf_counter() - start) * 1000
+        return TransformResult(
+            input_count=len(records),
+            output_count=len(current),
+            dropped_count=len(records) - len(current),
+            duration_ms=round(total_duration, 2),
+            records=current,
+            step_metrics=step_metrics,
+        )

dataenginex-0.3.4.dist-info/METADATA

@@ -0,0 +1,66 @@
+Metadata-Version: 2.4
+Name: dataenginex
+Version: 0.3.4
+Summary: DataEngineX - Core framework for data engineering projects
+License: MIT
+Author: Jay
+Author-email: jayapal.myaka99@gmail.com
+Requires-Python: >=3.11
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Requires-Dist: email-validator (>=2.0.0)
+Requires-Dist: fastapi (>=0.128.4)
+Requires-Dist: httpx (>=0.28.0)
+Requires-Dist: loguru (>=0.7.3)
+Requires-Dist: opentelemetry-api (>=1.39.0)
+Requires-Dist: opentelemetry-exporter-otlp (>=1.39.0)
+Requires-Dist: opentelemetry-instrumentation-fastapi (>=0.60b1)
+Requires-Dist: opentelemetry-sdk (>=1.39.0)
+Requires-Dist: prometheus-client (>=0.24.0)
+Requires-Dist: python-dotenv (>=1.2.0)
+Requires-Dist: python-json-logger (>=4.0.0)
+Requires-Dist: pyyaml (>=6.0.2)
+Requires-Dist: structlog (>=25.5.0)
+Requires-Dist: uvicorn (>=0.40.0)
+Description-Content-Type: text/markdown
+
+# dataenginex
+
+`dataenginex` is the core DataEngineX framework package for building observable, production-ready data and API services.
+
+It provides:
+- FastAPI application primitives and API extensions
+- Middleware for structured logging, metrics, and tracing
+- Data quality and validation utilities
+- Lakehouse and warehouse building blocks
+- Reusable ML support modules for model-serving workflows
+
+## Install
+
+```bash
+pip install dataenginex
+```
+
+## Package Scope
+
+This package is the core library from the DEX monorepo.
+`careerdex` and `weatherdex` are maintained in the same repository but are not part of this package release flow.
+
+## Quick Usage
+
+```python
+from dataenginex import __version__
+
+print(__version__)
+```
+
+## Source and Docs
+
+- Repository: https://github.com/data-literate/DEX
+- CI/CD guide: `docs/CI_CD.md`
+- Release notes: `packages/dataenginex/src/dataenginex/RELEASE_NOTES.md`
+

dataenginex-0.3.4.dist-info/RECORD

@@ -0,0 +1,41 @@
+dataenginex/README.md,sha256=CA8sEQ6QCFmmx5vE6lkRvGtf5N6zGG6aRi4vz7WpFsA,901
+dataenginex/RELEASE_NOTES.md,sha256=-jqu-JCcbRfa8Yry0qZ6yEl2nZ7P5PBzbYGpHj8GiNY,1812
+dataenginex/__init__.py,sha256=Y1UcBcb91_9R1emZaCoObxDovJkCo3e2umfjfMe6zZs,469
+dataenginex/api/__init__.py,sha256=z9YbGR8qY9qmJ4Gnx2mknT7UsncsUlG5aK5gbAYGM84,336
+dataenginex/api/auth.py,sha256=0MrtA6imcwAVynrXWUvEhQhaTuTocR8hxVc-FaSVBxw,5759
+dataenginex/api/errors.py,sha256=hDO4LsAiQqjyglccSj53FDSkLjo9lhjR2Twr9TDzvME,1843
+dataenginex/api/health.py,sha256=OTbY0kmy14KYoX463pFY0HYxhqtHfhZpwReY9GpIT20,4337
+dataenginex/api/pagination.py,sha256=dzHSXDXsjYiL8HzOXQR5wNTRYVqCzbCrb-ZE17EpWEQ,2693
+dataenginex/api/rate_limit.py,sha256=OajTPGiLCptoRnlH1-rX2cqTJjJJxe0QGd5R3wKumlI,4017
+dataenginex/api/routers/__init__.py,sha256=6pzSoxY7glXyKAfCChT-ui-N64qJkdEUlyR1ibN1Kyw,28
+dataenginex/api/routers/v1.py,sha256=HimY-vbmI8JoLSkQZxVqs5KGWsx_WLdUEp3xNGj6E9g,3831
+dataenginex/core/__init__.py,sha256=VMw1YlZoHSXarP5FyftrvgF0-7pJ-SUQvb3ZonlroZM,860
+dataenginex/core/medallion_architecture.py,sha256=Ai2wDxbPYAU4I8h61zk8skw7RkFLbZr5iLp7ZpXPXiI,14562
+dataenginex/core/pipeline_config.py,sha256=DM7v0-b4DBmXd8Bn4RqsQdWJlLsIxjytEBeasnYCmxQ,3860
+dataenginex/core/schemas.py,sha256=hyIvim0rRzAk0OceqXbAhMp0i8QM_AUq4zFzVy07Q7g,9599
+dataenginex/core/validators.py,sha256=btaOKhrAan_RWY7cD9Kdhr99EFz_khIqy7LKy-Et8dg,12872
+dataenginex/data/__init__.py,sha256=EyZ1_MRGQqbgE0Ss1EtJOa7n2Ga9n8LmPpb6d92XNaY,637
+dataenginex/data/connectors.py,sha256=5uQ2TAUGsoLwywnrhnNL-PFqpBZPcgZKlSninH9LI1M,10586
+dataenginex/data/profiler.py,sha256=PnYDavjHfSoBGInyd2WN_uFkASzTDTlbUHZIDrPABEc,7175
+dataenginex/data/registry.py,sha256=Ljlra7PV2-lq6vy7OXSs1vHkZ5rZ9Pm1JTLgv0FqYfk,5673
+dataenginex/lakehouse/__init__.py,sha256=lEEorD6i0cjZeGLcfDb89rFGRb3XvSDzbzIdrjgDf4Q,678
+dataenginex/lakehouse/catalog.py,sha256=UF9iAunQx9WVdtCzifoojs4RN9eQsgLSOaS_AmeU31g,5130
+dataenginex/lakehouse/partitioning.py,sha256=6pQcbALqB5gqyoT_HHtcTGq2EYh2FR1FVy5wiZ9z51Y,3444
+dataenginex/lakehouse/storage.py,sha256=wnzylQjeH8tZf0yb3DF0d4Bg2xVYzbVXrkQVDOd2VNI,6231
+dataenginex/middleware/__init__.py,sha256=ylCRjVgIzV8tusa6Ip8gTkbEpeyEdO7A_9Ud22uCzc4,621
+dataenginex/middleware/logging_config.py,sha256=QUnBxFtt3ROkuNR7OWxo_50HnSeV8Tn9dYUb36F2uX4,4578
+dataenginex/middleware/metrics.py,sha256=UKKZlK-vwOL3EA2QEkw5h65bUsQAD0spX_js8k76zUE,1028
+dataenginex/middleware/metrics_middleware.py,sha256=YnUaNUoMd48Q8q8yPylDfjDrTZPmeJqKLMimh7lp6mI,1879
+dataenginex/middleware/request_logging.py,sha256=kO7tSX-oAeBfVoCKwD4jV-TPYLqqsEJQZcOXu5Io9L4,2358
+dataenginex/middleware/tracing.py,sha256=bz4AttXSy75Rxf9SKkpomzKWPHnrD3-KKuWf_4SUE30,2575
+dataenginex/ml/__init__.py,sha256=_kGIalASJYSKosTOEySB7d17R1HryFmDOEGZM6AZ1o4,913
+dataenginex/ml/drift.py,sha256=2sY4xHZ4Mi9fOXiP-KmFvN9uBkxwWSpejTyg6ngMadY,5469
+dataenginex/ml/registry.py,sha256=dp1TN-saDnzPvDrhFIzvppphb2an97NPV7tu7yumC-I,5684
+dataenginex/ml/serving.py,sha256=WML6mJNaPuEy_tUr6ouiDK9EKX9CyP8BYcD1kS9CO7o,4636
+dataenginex/ml/training.py,sha256=-P72dZYD59IcY054rX4r0Z044yxpbfbIIdpZiET4TdI,6471
+dataenginex/warehouse/__init__.py,sha256=j3UTpKSVCA-JtFe_R_jlNCMhcsJLcT5wOWpCxm5Nghg,587
+dataenginex/warehouse/lineage.py,sha256=5oHlPnn7ogVaKpi7BKhBxXE7tLIqzE5xdW-nnTcUSnA,5570
+dataenginex/warehouse/transforms.py,sha256=5_rLWvsCOD-ZySEwyiRq0kkgj7KHMOK6EwPYYYiWPQc,7262
+dataenginex-0.3.4.dist-info/METADATA,sha256=EcWqEM6-XgsLFcJuAfcnBFNvrjXHjcnajs0b02kWx_M,2033
+dataenginex-0.3.4.dist-info/WHEEL,sha256=kJCRJT_g0adfAJzTx2GUMmS80rTJIVHRCfG0DQgLq3o,88
+dataenginex-0.3.4.dist-info/RECORD,,

dataenginex-0.3.4.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: poetry-core 2.3.1
+Root-Is-Purelib: true
+Tag: py3-none-any