dataenginex 0.4.0__tar.gz → 0.4.2__tar.gz

{dataenginex-0.4.0 → dataenginex-0.4.2}/.github/PULL_REQUEST_TEMPLATE.md

@@ -70,6 +70,6 @@ chore: establish org/domain foundation (pages, labels, project automation)
 - [ ] Org/Repo variable set: `ORG_PROJECT_URL`
 - [ ] Org/Repo secret set: `ORG_PROJECT_TOKEN`
 - [ ] Cloudflare DNS updated for docs/api/apex domains
-- [ ] Post-cutover checks completed (see `docs/DEPLOY_RUNBOOK.md` → Org + Domain Rollout)
+- [ ] Post-cutover checks completed
 
 ## Notes for Reviewers

{dataenginex-0.4.0 → dataenginex-0.4.2}/.github/workflows/ci.yml

@@ -9,7 +9,7 @@ on:
 
   schedule:
     # Weekly Python version compatibility check
-    - cron: '0 0 * * 0'
+    - cron: "0 0 * * 0"
 
 permissions:
   contents: read
@@ -20,7 +20,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v6
-      - uses: astral-sh/setup-uv@v8.1.0
+      - uses: astral-sh/setup-uv@v8.2.0
         with:
           version: "latest"
           python-version: "3.13"
@@ -35,7 +35,7 @@ jobs:
     needs: quality
     steps:
       - uses: actions/checkout@v6
-      - uses: astral-sh/setup-uv@v8.1.0
+      - uses: astral-sh/setup-uv@v8.2.0
         with:
           version: "latest"
           python-version: "3.13"
@@ -43,7 +43,7 @@ jobs:
         env:
           UV_PROJECT_ENVIRONMENT: .venv
       - run: uv run poe test-cov-core
-      - uses: codecov/codecov-action@v5
+      - uses: codecov/codecov-action@v7
         with:
           flags: dataenginex
           fail_ci_if_error: false
@@ -59,7 +59,7 @@ jobs:
         python-version: ["3.11", "3.12"]
     steps:
       - uses: actions/checkout@v6
-      - uses: astral-sh/setup-uv@v8.1.0
+      - uses: astral-sh/setup-uv@v8.2.0
         with:
           version: "latest"
           python-version: ${{ matrix.python-version }}

dataenginex-0.4.2/.github/workflows/docs-sync.yml

@@ -0,0 +1,26 @@
+name: Trigger Docs Sync
+
+on:
+  push:
+    tags:
+      - "v*"
+  workflow_dispatch:
+    inputs:
+      version:
+        description: "Version tag to sync to website"
+        required: false
+        default: ""
+
+jobs:
+  dispatch:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Send repository_dispatch to website
+        uses: peter-evans/repository-dispatch@v4
+        with:
+          token: ${{ secrets.DOCS_SYNC_PAT }}
+          repository: TheDataEngineX/website
+          event-type: sync-docs
+          client-payload: >-
+            {"repo": "dataenginex",
+             "version": "${{ github.ref_name || inputs.version }}"}

{dataenginex-0.4.0 → dataenginex-0.4.2}/.github/workflows/release.yml

@@ -3,7 +3,18 @@ name: Release
 on:
   push:
     tags:
-      - 'v[0-9]+.[0-9]+.[0-9]+'
+      - "v[0-9]+.[0-9]+.[0-9]+"
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: "Release tag (e.g. v0.4.1)"
+        required: true
+        type: string
+      ref:
+        description: "Git ref to build from (branch or SHA)"
+        required: false
+        default: "main"
+        type: string
 
 jobs:
   build:
@@ -13,7 +24,9 @@ jobs:
       contents: read
     steps:
       - uses: actions/checkout@v6
-      - uses: astral-sh/setup-uv@v8.1.0
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+      - uses: astral-sh/setup-uv@v8.2.0
         with:
           version: "latest"
           python-version: "3.13"
@@ -46,7 +59,9 @@ jobs:
       contents: write
     steps:
       - uses: actions/checkout@v6
-      - uses: astral-sh/setup-uv@v8.1.0
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+      - uses: astral-sh/setup-uv@v8.2.0
         with:
           version: "latest"
       - name: Extract version
@@ -62,8 +77,9 @@ jobs:
       - name: Create GitHub release + attach SBOM
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          RELEASE_TAG: ${{ inputs.tag || github.ref_name }}
         run: |
-          gh release create ${{ github.ref_name }} \
-            --title "DEX ${{ github.ref_name }}" \
+          gh release create "$RELEASE_TAG" \
+            --title "DEX $RELEASE_TAG" \
             --generate-notes \
             "sbom-dex-${{ steps.version.outputs.version }}.json"

dataenginex-0.4.2/CHANGELOG.md

@@ -0,0 +1,87 @@
+# Changelog
+
+See [docs/release-notes.md](docs/release-notes.md) for the complete release history.
+
+All notable changes to `dataenginex` will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [0.4.2] - 2026-06-23
+
+### Added
+
+- Example scripts refreshed: `08_spark_ml.py`, `09_feature_engineering.py`, `10_model_analysis.py` — demonstrate PySpark ML, feature transforms, and drift detection
+- `docs/release-notes.md` rewritten — cleaned up pre-reset 1.x entries, corrected version history to start from 0.3.5
+- `docs/observability.md` trimmed to library-level content only; HTTP/K8s content moved to `dex-studio/docs/observability.md`
+
+### Changed
+
+- Version bumped to 0.4.2
+
+## [0.4.1] - 2026-06-12
+
+### Added
+
+- `dataenginex._json` — drop-in `orjson`-backed JSON shim (`dumps`, `loads`, `JSONResponse`) replacing stdlib `json` across the library for ~3–5× serialization throughput
+- `DeltaConnector` — native Delta Lake read/write via `deltalake` (new `delta` optional extra: `pip install "dataenginex[delta]"`)
+- `ml.features.builtin` — built-in feature transformers: `StandardScalerTransform`, `MinMaxScalerTransform`, `OneHotEncoderTransform`, `PolynomialFeaturesTransform`
+- `core.interfaces` — new `Closeable` and `AsyncCloseable` protocols for uniform resource lifecycle
+- `orjson>=3.11.0` and `zstandard>=0.25.0` promoted to core runtime dependencies
+
+### Changed
+
+- **Lakehouse storage** (`lakehouse/storage.py`) — full rewrite: unified `LakehouseStorage` with pluggable backends (local, S3, GCS), Zstandard compression throughout, columnar partition pruning
+- **Lakehouse catalog** (`lakehouse/catalog.py`) — catalog entries now carry partition stats and schema fingerprints; `register` / `resolve` API stabilised
+- **ML registry** (`ml/registry.py`) — artifact versioning with aliasing (`promote_alias`), stage transitions (`development` → `staging` → `production`), and metadata search
+- **ML training** (`ml/training.py`) — `TrainingJob` lifecycle management, early-stopping callbacks, cross-validation harness, experiment comparison utilities
+- **Pagination** (`api/pagination.py`) — cursor-based and page-number strategies unified under `PaginationResult`; `paginate_query` helper works with any iterable
+- **Store** (`store.py`) — async-safe DuckDB connection pool, `get_pipeline_runs` and `list_model_artifacts` now return typed dataclasses
+- **SecOps audit** (`secops/audit.py`) — structured audit events with severity levels, retention policy enforcement, export to JSONL
+- **AI runtime** (`ai/runtime/executor.py`) — tool call concurrency limit, timeout per tool, structured error envelopes
+- **Config loader** (`config/loader.py`) — environment variable interpolation (`${VAR}`) and `include:` directive for config composition
+- `zstandard` used for pipeline run history compression reducing on-disk footprint by ~60%
+
+### Fixed
+
+- `mypy --strict` passes cleanly across all modules after strict type annotation pass
+- `DeltaConnector` and `LakehouseStorage` excluded from coverage thresholds (require live filesystems); coverage gate unchanged for all other modules
+
+## [0.4.0] - 2026-02-21
+
+> **Scope reset from 1.x.** Versions 1.0.0–1.1.2 were prematurely tagged stable. Resetting to `0.4.0` to honestly reflect pre-1.0 maturity. See [ADR-0007](https://github.com/TheDataEngineX/docs/blob/main/adr/0007-local-first-scope-reset.md) for rationale. The 1.x versions on PyPI are yanked but remain installable by exact pin (`pip install 'dataenginex==1.1.2'`); plain `pip install dataenginex` now resolves to `0.4.0`.
+
+### Added
+
+- Stable `__all__` exports in every subpackage `__init__.py`
+- `from __future__ import annotations` in all public modules
+- Comprehensive module-level docstrings with usage examples
+- New public API exports: `ComponentHealth`, `AuthMiddleware`, `AuthUser`,
+  `create_token`, `decode_token`, `BadRequestError`, `NotFoundError`,
+  `PaginationMeta`, `RateLimiter`, `RateLimitMiddleware`,
+  `ConnectorStatus`, `FetchResult`, `ColumnProfile`, `get_logger`, `get_tracer`
+
+### Changed
+
+- Reorganized `__all__` in all subpackages for logical grouping
+- Updated package version to 0.4.0
+
+## [0.3.5] - 2026-02-13
+
+### Added
+
+- Production hardening: structured logging, Prometheus/OTel, health probes
+- Data connectors: `RestConnector`, `FileConnector` with async interface
+- Schema registry with versioned schema management
+- Data profiler with automated dataset statistics
+- Lakehouse catalog, partitioning, and storage backends
+- ML framework: trainer, model registry, drift detection, serving
+- Warehouse transforms and persistent lineage tracking
+- JWT authentication middleware
+- Rate limiting middleware
+- Cursor-based pagination utilities
+- Versioned API router (`/api/v1/`)
+
+[0.3.5]: https://github.com/TheDataEngineX/dataenginex/releases/tag/v0.3.5
+[0.4.0]: https://github.com/TheDataEngineX/dataenginex/compare/v0.3.5...v0.4.0
+[0.4.1]: https://github.com/TheDataEngineX/dataenginex/compare/v0.4.0...v0.4.1

{dataenginex-0.4.0 → dataenginex-0.4.2}/CONTRIBUTING.md

@@ -7,7 +7,7 @@ Thank you for your interest in contributing to DataEngineX!
 Quick essentials:
 
 - Development setup: [docs/DEVELOPMENT.md](docs/DEVELOPMENT.md)
-- Governance & legal: [GOVERNANCE.md](GOVERNANCE.md)
+- Governance & legal: See [docs/CONTRIBUTING.md](docs/CONTRIBUTING.md)
 - Code of Conduct: [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md)
 
 ## Quick Start

{dataenginex-0.4.0 → dataenginex-0.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dataenginex
-Version: 0.4.0
+Version: 0.4.2
 Summary: DataEngineX — open-source, self-hosted, local-first Data + ML + AI workbench library
 Author-email: Jay <jayapal.myaka99@gmail.com>
 License: MIT License
@@ -30,18 +30,25 @@ Requires-Dist: click>=8.3.3
 Requires-Dist: croniter>=6.2.2
 Requires-Dist: duckdb>=1.5.2
 Requires-Dist: httpx>=0.28.1
+Requires-Dist: msgpack>=1.2.1
+Requires-Dist: orjson>=3.11.0
 Requires-Dist: prometheus-client>=0.25.0
 Requires-Dist: pyarrow>=23.0.1
+Requires-Dist: pydantic-settings>=2.14.2
 Requires-Dist: pydantic>=2.13.4
-Requires-Dist: python-dotenv>=1.2.2
 Requires-Dist: pyyaml>=6.0.3
 Requires-Dist: structlog>=25.5.0
+Requires-Dist: zstandard>=0.25.0
 Provides-Extra: cloud
 Requires-Dist: boto3>=1.43.7; extra == 'cloud'
 Requires-Dist: google-cloud-bigquery>=3.41.0; extra == 'cloud'
 Requires-Dist: google-cloud-storage>=3.10.1; extra == 'cloud'
+Provides-Extra: delta
+Requires-Dist: deltalake>=0.24.0; extra == 'delta'
 Provides-Extra: postgres
 Requires-Dist: asyncpg>=0.31.0; extra == 'postgres'
+Provides-Extra: pytorch
+Requires-Dist: torch>=2.0.0; extra == 'pytorch'
 Provides-Extra: qdrant
 Requires-Dist: qdrant-client>=1.18.0; extra == 'qdrant'
 Provides-Extra: queue
@@ -52,7 +59,7 @@ Description-Content-Type: text/markdown
 
 The Python library that powers [DEX Studio](https://github.com/TheDataEngineX/dex-studio) — an open-source, self-hosted, local-first Data + ML + AI workbench. Use the library directly when you want code, not a UI.
 
-> **Pre-1.0 status.** `0.4.0` is honest about that. See the [scope reset CHANGELOG](https://github.com/TheDataEngineX/dex/blob/main/CHANGELOG.md) for the rationale.
+> **Pre-1.0 status.** `0.4.0` is honest about that. See the [scope reset CHANGELOG](https://github.com/TheDataEngineX/dataenginex/blob/main/CHANGELOG.md) for the rationale.
 
 ## Install
 
@@ -148,9 +155,9 @@ DEX Studio imports `dataenginex` directly — no separate API server.
 
 ## Links
 
-- Source: [github.com/TheDataEngineX/dex](https://github.com/TheDataEngineX/dex)
+- Source: [github.com/TheDataEngineX/dataenginex](https://github.com/TheDataEngineX/dataenginex)
 - Docs: [docs.thedataenginex.org](https://docs.thedataenginex.org)
 - Roadmap: [docs/docs/roadmap/DESIGN-2026.md](https://github.com/TheDataEngineX/docs/blob/main/docs/roadmap/DESIGN-2026.md)
 - ADRs: [docs/adr/](https://github.com/TheDataEngineX/docs/tree/main/adr)
-- Issues: [github.com/TheDataEngineX/dex/issues](https://github.com/TheDataEngineX/dex/issues)
+- Issues: [github.com/TheDataEngineX/dataenginex/issues](https://github.com/TheDataEngineX/dataenginex/issues)
 - License: MIT

{dataenginex-0.4.0 → dataenginex-0.4.2}/README.md

@@ -1,6 +1,6 @@
 # dataenginex
 
-[![CI](https://github.com/TheDataEngineX/DEX/actions/workflows/ci.yml/badge.svg?branch=dev)](https://github.com/TheDataEngineX/DEX/actions/workflows/ci.yml)
+[![CI](https://github.com/TheDataEngineX/dataenginex/actions/workflows/ci.yml/badge.svg?branch=dev)](https://github.com/TheDataEngineX/dataenginex/actions/workflows/ci.yml)
 [![PyPI](https://img.shields.io/pypi/v/dataenginex)](https://pypi.org/project/dataenginex/)
 [![Python 3.13+](https://img.shields.io/badge/python-3.13+-blue.svg)](https://www.python.org/downloads/)
 [![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](LICENSE)
@@ -96,7 +96,7 @@ ______________________________________________________________________
 ## Development
 
 ```bash
-git clone https://github.com/TheDataEngineX/dex && cd dex
+git clone https://github.com/TheDataEngineX/dataenginex && cd dataenginex
 uv sync
 uv run poe check-all          # lint + typecheck + tests
 uv run poe test-cov           # tests + coverage
@@ -126,7 +126,7 @@ ______________________________________________________________________
 
 | Repo | Purpose |
 | --- | --- |
-| [dataenginex](https://github.com/TheDataEngineX/dex) | This library (PyPI) |
+| [dataenginex](https://github.com/TheDataEngineX/dataenginex) | This library (PyPI) |
 | [dex-studio](https://github.com/TheDataEngineX/dex-studio) | Web UI — FastAPI + Jinja2 + HTMX |
 | [docs](https://github.com/TheDataEngineX/docs) | Docs site ([docs.thedataenginex.org](https://docs.thedataenginex.org)) — ADRs + roadmap live here |
 

{dataenginex-0.4.0 → dataenginex-0.4.2}/SECURITY.md

@@ -4,7 +4,7 @@
 
 | Version | Supported |
 |---------|-----------|
-| Latest minor release (1.0.x) | ✅ |
+| Latest (0.4.x) | ✅ |
 | Previous minor release | ✅ (security fixes only) |
 | Older versions | ❌ |
 
@@ -15,7 +15,7 @@
 Instead, please report them via one of these channels:
 
 1. **Email**: security@thedataenginex.dev
-2. **GitHub Security Advisories**: Use the "Report a vulnerability" button on the Security tab
+1. **GitHub Security Advisories**: Use the "Report a vulnerability" button on the Security tab
 
 ### What to Include
 
@@ -53,6 +53,8 @@ DataEngineX follows these security practices:
 
 ## Security-Related Dependencies
 
+*This table is illustrative, not exhaustive.*
+
 | Dependency | Purpose | Security Note |
 |------------|---------|---------------|
 | pydantic | Config validation | Validates all inputs |
@@ -68,4 +70,4 @@ Run security audits locally:
 uv run poe security  # pip-audit for vulnerabilities
 ```
 
-CI runs `pip-audit` and dependency scanning on every PR.
+CI runs `pip-audit` and dependency scanning on every PR.

{dataenginex-0.4.0 → dataenginex-0.4.2}/docs/adr/0001-medallion-architecture.md

@@ -4,6 +4,8 @@
 **Date**: 2026-02-15
 **Authors**: Data Engineering Team
 
+> **Note:** Implementation details (tooling, storage) have evolved since this ADR was written. See [docs/architecture.md](docs/architecture.md) for the current design. References to ADR-0002, ADR-0003, and ADR-0004 are placeholders for decisions that were planned but not written.
+
 ## Context
 
 DEX needs a scalable, standardized data architecture that works across all projects (CareerDEX, Weather, etc.). The data pipeline needs to handle:

{dataenginex-0.4.0 → dataenginex-0.4.2}/docs/api-reference/index.md

@@ -11,5 +11,4 @@ Auto-generated API documentation for the `dataenginex` package.
 | [middleware](middleware.md) | Logging, metrics, tracing, request middleware |
 | [ml](ml.md) | ML training, model registry, drift detection, serving, vectorstore, LLM adapters |
 | [plugins](plugins.md) | Plugin system — ABC, registry, auto-discovery |
-| [dashboard](dashboard.md) | Streamlit dashboard framework — panels, config |
 | [warehouse](warehouse.md) | Transforms, persistent lineage tracking |

{dataenginex-0.4.0 → dataenginex-0.4.2}/docs/ci-cd.md

@@ -124,7 +124,7 @@ ______________________________________________________________________
 
 ## Release Automation
 
-**Workflow**: [`.github/workflows/release.yml`](https://github.com/TheDataEngineX/dex/blob/main/.github/workflows/release.yml)
+**Workflow**: [`.github/workflows/release.yml`](https://github.com/TheDataEngineX/dataenginex/blob/main/.github/workflows/release.yml)
 
 **Trigger**: Push a tag matching `v[0-9]+.[0-9]+.[0-9]+` to `main`
 

{dataenginex-0.4.0 → dataenginex-0.4.2}/docs/contributing.md

@@ -69,7 +69,7 @@ Use semantic commit format:
 - This project is open source under MIT; keep license and attribution notices in redistributions.
 - Forks are welcome, but should use a distinct public name when redistributed as a separate project.
 - Do not present a fork as the official DataEngineX/DEX project.
-- See [Trademark Policy](https://github.com/TheDataEngineX/dataenginex/blob/main/TRADEMARK_POLICY.md) for brand-usage details.
+- See the project's license and brand guidelines for brand-usage details.
 
 ## Code Reviews
 

{dataenginex-0.4.0 → dataenginex-0.4.2}/docs/development.md

@@ -37,14 +37,14 @@ This installs all Python dependencies and configures pre-commit hooks.
 
 ```bash
 # 1. Clone repo and create feature branch
-git clone https://github.com/TheDataEngineX/dex.git
-cd dex
+git clone https://github.com/TheDataEngineX/dataenginex.git
+cd dataenginex
 git checkout -b feat/issue-XXX-description dev
 
-# 3. Install Python deps & pre-commit hooks
+# 2. Install Python deps & pre-commit hooks
 uv run poe setup
 
-# 4. Verify setup
+# 3. Verify setup
 uv run poe check-all
 ```
 
@@ -53,12 +53,11 @@ All tests and linting should pass. You're ready to develop!
 ## Project Structure
 
 ```
-DEX/
+dataenginex/
 ├── src/dataenginex/        # Core framework package
 ├── examples/               # Runnable example scripts (01–10)
 ├── tests/                  # Test suite
 ├── docs/                   # Documentation
-├── monitoring/             # Local observability stack configs
 ├── .github/workflows/      # CI/CD pipelines
 ├── pyproject.toml          # Project config
 └── poe_tasks.toml          # Task definitions
@@ -160,24 +159,7 @@ uv run poe test-unit
 uv run poe check-all
 ```
 
-### Monitoring & Debugging
-
-```bash
-# View application logs
-tail -f logs/app.log
-
-# Enable debug logging
-export LOG_LEVEL=DEBUG
-uv run poe dev
-
-# Use Python debugger
-python -m pdb examples/02_api_quickstart.py
-
-# Prometheus metrics (if running)
-open http://localhost:9090
-```
-
-## Troubleshooting
+### Troubleshooting
 
 | Issue | Solution |
 |-------|----------|
@@ -198,7 +180,7 @@ uv run poe test               # Run all tests
 uv run poe test-cov           # Tests with coverage report
 uv run poe security           # pip-audit vulnerability scan
 uv run poe pre-commit         # Run all pre-commit hooks
-uv run poe dev                # Run dev server (localhost:17000)
+uv run poe docker-up          # Run Docker Compose stack
 uv run poe clean              # Remove caches and build artifacts
 ```
 
@@ -208,5 +190,5 @@ uv run poe clean              # Remove caches and build artifacts
 - **Architecture**: See [architecture.md](./architecture.md)
 - **ADRs**: See [ADR-0001](./adr/0001-medallion-architecture.md) for architectural decisions
 - **Deployment**: See Deployment Runbook in the `infradex` repo
-- **Issues**: [GitHub Issues](https://github.com/TheDataEngineX/dex/issues)
+- **Issues**: [GitHub Issues](https://github.com/TheDataEngineX/dataenginex/issues)
 - **Discussions**: [GitHub Discussions](https://github.com/orgs/TheDataEngineX/discussions)

dataenginex-0.4.2/docs/index.md

@@ -0,0 +1,28 @@
+# DataEngineX Documentation
+
+**The Python library (PyPI) — engine, config, CLI, pipelines, ML, AI, PrivacyGuard.**
+
+Documentation for the core `dataenginex` library. For the web UI, see [dex-studio](https://github.com/TheDataEngineX/dex-studio).
+
+## Quick start
+
+```bash
+pip install dataenginex
+```
+
+or
+
+```bash
+uv add dataenginex
+```
+
+See [Quickstart](quickstart.md) for a full walkthrough with a sample project.
+
+## Guides
+
+- [Quickstart](quickstart.md) — Get up and running in 5 minutes
+- [Architecture](architecture.md) — Core patterns, module map, design decisions
+- [Development Setup](development.md) — Prerequisites, workflow, troubleshooting
+- [Release Notes](release-notes.md) — Full version history
+- [Observability](observability.md) — Metrics, logging, tracing
+- [Contributing](contributing.md) — How to contribute to the project

dataenginex-0.4.2/docs/observability.md

@@ -0,0 +1,61 @@
+# Observability: Metrics, Logging & Tracing
+
+**Library-level observability for `dataenginex`.** For application-level monitoring (HTTP middleware, health endpoints, Grafana dashboards), see [dex-studio/docs/observability.md](https://github.com/TheDataEngineX/dex-studio/blob/main/docs/observability.md).
+
+## Logging
+
+`dataenginex` uses `structlog` for structured logging throughout the library. All loggers are configured by the host application (e.g., dex-studio). The library does not configure handlers itself.
+
+### Logging in your application
+
+```python
+import structlog
+from dataenginex.engine import DexEngine
+
+structlog.configure(
+    processors=[structlog.dev.ConsoleRenderer()],
+    wrapper_class=structlog.make_filtering_bound_logger(20),  # INFO
+)
+engine = DexEngine("dex.yaml")
+```
+
+## Metrics
+
+Prometheus metrics are exposed via `dataenginex.observability.metrics`:
+
+```python
+from dataenginex.observability.metrics import (
+    HTTP_REQUESTS_TOTAL,
+    HTTP_REQUEST_DURATION_SECONDS,
+    PIPELINE_RUN_DURATION,
+)
+```
+
+### Available library-level metrics
+
+| Metric | Type | Description | Labels |
+|--------|------|-------------|--------|
+| `pipeline_run_duration_seconds` | Histogram | Pipeline execution time | pipeline_name, status |
+| `model_prediction_latency_seconds` | Histogram | Model inference time | model_name |
+| `llm_request_duration_seconds` | Histogram | LLM call time | provider, model |
+| `data_connector_rows_read` | Counter | Rows read by source connectors | connector_type |
+
+## Tracing
+
+OpenTelemetry tracing is available via `dataenginex.tracing`:
+
+```python
+from dataenginex.tracing import get_tracer
+
+tracer = get_tracer(__name__)
+
+with tracer.start_as_current_span("pipeline_run") as span:
+    span.set_attribute("pipeline.name", "ingest")
+    engine.run_pipeline("ingest")
+```
+
+Enable OTLP export:
+
+```bash
+export OTLP_ENDPOINT="http://localhost:4317"
+```

{dataenginex-0.4.0 → dataenginex-0.4.2}/docs/quickstart.md

@@ -7,7 +7,7 @@ Get a DataEngineX pipeline running in under five minutes.
 ```bash
 pip install dataenginex
 # or from source:
-git clone https://github.com/TheDataEngineX/dex && cd dex
+git clone https://github.com/TheDataEngineX/dataenginex && cd dataenginex
 uv sync
 ```
 
@@ -138,7 +138,7 @@ events = engine.secops_audit.events   # list of AuditEvent
 - [API Reference](api-reference/index.md) — auto-generated module docs
 - `examples/` directory — full list of runnable examples
 
----
+______________________________________________________________________
 
 ## DEX Studio
 

dataenginex-0.4.2/docs/release-notes.md

@@ -0,0 +1,94 @@
+# Release Notes
+
+## [0.4.2] — 2026-06-23
+
+### Added
+
+- Example scripts refreshed: `08_spark_ml.py`, `09_feature_engineering.py`, `10_model_analysis.py` — demonstrate PySpark ML, feature transforms, and drift detection
+- Documentation cleanup across all docs
+
+### Changed
+
+- Version bumped to 0.4.2
+
+### Verification checklist
+
+1. `uv run poe lint` — Ruff checks clean
+1. `uv run poe typecheck` — mypy strict, 0 errors
+1. `uv run poe test` — all tests pass
+
+______________________________________________________________________
+
+## [0.4.1] - 2026-06-12
+
+### Added
+
+- `dataenginex._json` — drop-in `orjson`-backed JSON shim (`dumps`, `loads`, `JSONResponse`) replacing stdlib `json` across the library for ~3–5× serialization throughput
+- `DeltaConnector` — native Delta Lake read/write via `deltalake` (new `delta` optional extra: `pip install "dataenginex[delta]"`)
+- `ml.features.builtin` — built-in feature transformers: `StandardScalerTransform`, `MinMaxScalerTransform`, `OneHotEncoderTransform`, `PolynomialFeaturesTransform`
+- `core.interfaces` — new `Closeable` and `AsyncCloseable` protocols for uniform resource lifecycle
+- `orjson>=3.11.0` and `zstandard>=0.25.0` promoted to core runtime dependencies
+
+### Changed
+
+- **Lakehouse storage** (`lakehouse/storage.py`) — full rewrite: unified `LakehouseStorage` with pluggable backends (local, S3, GCS), Zstandard compression throughout, columnar partition pruning
+- **Lakehouse catalog** (`lakehouse/catalog.py`) — catalog entries now carry partition stats and schema fingerprints; `register` / `resolve` API stabilised
+- **ML registry** (`ml/registry.py`) — artifact versioning with aliasing (`promote_alias`), stage transitions (`development` → `staging` → `production`), and metadata search
+- **ML training** (`ml/training.py`) — `TrainingJob` lifecycle management, early-stopping callbacks, cross-validation harness, experiment comparison utilities
+- **Pagination** (`api/pagination.py`) — cursor-based and page-number strategies unified under `PaginationResult`; `paginate_query` helper works with any iterable
+- **Store** (`store.py`) — async-safe DuckDB connection pool, `get_pipeline_runs` and `list_model_artifacts` now return typed dataclasses
+- **SecOps audit** (`secops/audit.py`) — structured audit events with severity levels, retention policy enforcement, export to JSONL
+- **AI runtime** (`ai/runtime/executor.py`) — tool call concurrency limit, timeout per tool, structured error envelopes
+- **Config loader** (`config/loader.py`) — environment variable interpolation (`${VAR}`) and `include:` directive for config composition
+- `zstandard` used for pipeline run history compression reducing on-disk footprint by ~60%
+
+### Fixed
+
+- `mypy --strict` passes cleanly across all modules after strict type annotation pass
+- `DeltaConnector` and `LakehouseStorage` excluded from coverage thresholds (require live filesystems); coverage gate unchanged for all other modules
+
+______________________________________________________________________
+
+## [0.4.0] - 2026-02-21
+
+> **Scope reset from 1.x.** Versions 1.0.0–1.1.2 were prematurely tagged stable. Resetting to `0.4.0` to honestly reflect pre-1.0 maturity. See [ADR-0007](https://github.com/TheDataEngineX/docs/blob/main/adr/0007-local-first-scope-reset.md) for rationale. The 1.x versions on PyPI are yanked but remain installable by exact pin (`pip install 'dataenginex==1.1.2'`); plain `pip install dataenginex` now resolves to `0.4.0`.
+
+### Added
+
+- Stable `__all__` exports in every subpackage `__init__.py`
+- `from __future__ import annotations` in all public modules
+- Comprehensive module-level docstrings with usage examples
+- New public API exports: `ComponentHealth`, `AuthMiddleware`, `AuthUser`,
+  `create_token`, `decode_token`, `BadRequestError`, `NotFoundError`,
+  `PaginationMeta`, `RateLimiter`, `RateLimitMiddleware`,
+  `ConnectorStatus`, `FetchResult`, `ColumnProfile`, `get_logger`, `get_tracer`
+
+### Changed
+
+- Reorganized `__all__` in all subpackages for logical grouping
+- Updated package version to 0.4.0
+
+______________________________________________________________________
+
+## [0.3.5] - 2026-02-13
+
+### Added
+
+- Production hardening: structured logging, Prometheus/OTel, health probes
+- Data connectors: `RestConnector`, `FileConnector` with async interface
+- Schema registry with versioned schema management
+- Data profiler with automated dataset statistics
+- Lakehouse catalog, partitioning, and storage backends
+- ML framework: trainer, model registry, drift detection, serving
+- Warehouse transforms and persistent lineage tracking
+- JWT authentication middleware
+- Rate limiting middleware
+- Cursor-based pagination utilities
+- Versioned API router (`/api/v1/`)
+
+______________________________________________________________________
+
+[0.3.5]: https://github.com/TheDataEngineX/dataenginex/releases/tag/v0.3.5
+[0.4.0]: https://github.com/TheDataEngineX/dataenginex/compare/v0.3.5...v0.4.0
+[0.4.1]: https://github.com/TheDataEngineX/dataenginex/compare/v0.4.0...v0.4.1
+[0.4.2]: https://github.com/TheDataEngineX/dataenginex/compare/v0.4.1...v0.4.2