dataenginex 1.1.1__tar.gz → 1.1.2__tar.gz

{dataenginex-1.1.1 → dataenginex-1.1.2}/.github/workflows/ci.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v6
-      - uses: astral-sh/setup-uv@v7
+      - uses: astral-sh/setup-uv@v8.1.0
         with:
           version: "latest"
           python-version: "3.13"
@@ -35,7 +35,7 @@ jobs:
     needs: quality
     steps:
       - uses: actions/checkout@v6
-      - uses: astral-sh/setup-uv@v7
+      - uses: astral-sh/setup-uv@v8.1.0
         with:
           version: "latest"
           python-version: "3.13"
@@ -43,7 +43,7 @@ jobs:
         env:
           UV_PROJECT_ENVIRONMENT: .venv
       - run: uv run poe test-cov-core
-      - uses: codecov/codecov-action@v6
+      - uses: codecov/codecov-action@v5
         with:
           flags: dataenginex
           fail_ci_if_error: false
@@ -59,7 +59,7 @@ jobs:
         python-version: ["3.11", "3.12"]
     steps:
       - uses: actions/checkout@v6
-      - uses: astral-sh/setup-uv@v7
+      - uses: astral-sh/setup-uv@v8.1.0
         with:
           version: "latest"
           python-version: ${{ matrix.python-version }}

{dataenginex-1.1.1 → dataenginex-1.1.2}/.github/workflows/release.yml

@@ -13,12 +13,12 @@ jobs:
       contents: read
     steps:
       - uses: actions/checkout@v6
-      - uses: astral-sh/setup-uv@v7
+      - uses: astral-sh/setup-uv@v8.1.0
         with:
           version: "latest"
           python-version: "3.13"
       - run: uv build
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v7
         with:
           name: dist
           path: dist/
@@ -32,7 +32,7 @@ jobs:
       contents: read
       id-token: write
     steps:
-      - uses: actions/download-artifact@v4
+      - uses: actions/download-artifact@v8
         with:
           name: dist
           path: dist/
@@ -46,7 +46,7 @@ jobs:
       contents: write
     steps:
       - uses: actions/checkout@v6
-      - uses: astral-sh/setup-uv@v7
+      - uses: astral-sh/setup-uv@v8.1.0
         with:
           version: "latest"
       - name: Extract version

{dataenginex-1.1.1 → dataenginex-1.1.2}/.gitignore

@@ -65,6 +65,7 @@ desktop.ini
 # Logs and databases
 *.log
 *.sqlite3
+*.duckdb
 logs/
 .dex/
 

{dataenginex-1.1.1 → dataenginex-1.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dataenginex
-Version: 1.1.1
+Version: 1.1.2
 Summary: DataEngineX - Core framework for AI/ML/Data engineering projects
 Author-email: Jay <jayapal.myaka99@gmail.com>
 License: MIT License
@@ -31,27 +31,17 @@ Requires-Dist: asyncpg>=0.31.0
 Requires-Dist: click>=8.3.3
 Requires-Dist: croniter>=6.2.2
 Requires-Dist: duckdb>=1.5.2
-Requires-Dist: email-validator>=2.3.0
-Requires-Dist: fastapi>=0.136.1
 Requires-Dist: httpx>=0.28.1
-Requires-Dist: opentelemetry-api>=1.41.1
-Requires-Dist: opentelemetry-exporter-otlp>=1.41.1
-Requires-Dist: opentelemetry-instrumentation-fastapi>=0.62b1
-Requires-Dist: opentelemetry-sdk>=1.41.1
 Requires-Dist: prometheus-client>=0.25.0
 Requires-Dist: pyarrow>=23.0.1
 Requires-Dist: pydantic>=2.13.4
 Requires-Dist: python-dotenv>=1.2.2
 Requires-Dist: pyyaml>=6.0.3
-Requires-Dist: qdrant-client>=1.17.1
+Requires-Dist: qdrant-client>=1.18.0
 Requires-Dist: redis>=5.3.1
 Requires-Dist: structlog>=25.5.0
-Requires-Dist: uvicorn>=0.46.0
-Provides-Extra: auth
-Requires-Dist: ldap3>=2.9.1; extra == 'auth'
-Requires-Dist: pyjwt[crypto]>=2.12.1; extra == 'auth'
 Provides-Extra: cloud
-Requires-Dist: boto3>=1.43.4; extra == 'cloud'
+Requires-Dist: boto3>=1.43.7; extra == 'cloud'
 Requires-Dist: google-cloud-bigquery>=3.41.0; extra == 'cloud'
 Requires-Dist: google-cloud-storage>=3.10.1; extra == 'cloud'
 Provides-Extra: observability
@@ -60,44 +50,72 @@ Description-Content-Type: text/markdown
 
 # dataenginex
 
-Unified Data + ML + AI framework. Config-driven, self-hosted, production-ready.
+Unified Data + ML + AI **library**. Config-driven, self-hosted, production-ready.
+
+`dataenginex` is a pure Python library — no HTTP server. Your application owns the server layer.
 
 ## Install
 
 ```bash
-# Core (DuckDB, FastAPI, structlog, Pydantic, Click, Rich)
+# Core (DuckDB, structlog, Pydantic, Click, arq, asyncpg, qdrant-client)
 pip install dataenginex
 
-# With optional extras
-pip install dataenginex[dagster]      # Dagster orchestration
-pip install dataenginex[mlflow]       # MLflow experiment tracking
-pip install dataenginex[agents]       # LangGraph agent runtime
-pip install dataenginex[vectors]      # Qdrant + LanceDB vector stores
-pip install dataenginex[embeddings]   # sentence-transformers + ONNX
-pip install dataenginex[spark]        # PySpark transforms
-pip install dataenginex[cloud]        # S3 + GCS storage backends
-pip install dataenginex[all]          # Everything
+# Optional extras
+pip install "dataenginex[cloud]"          # S3 + GCS + BigQuery storage backends
+pip install "dataenginex[observability]"  # Langfuse LLM call tracing
 ```
 
+> **LiteLLM:** Install separately — it pins `python-dotenv==1.0.1` which conflicts
+> with dataenginex's `python-dotenv>=1.2.2`.
+> ```bash
+> pip install 'litellm>=1.83.3' --no-deps
+> ```
+
 ## Submodules
 
-| Module | Requires Extra | Description |
-|--------|---------------|-------------|
-| `dataenginex.config` | — | dex.yaml schema, loader, env var resolution, layering |
-| `dataenginex.core` | — | Exceptions, interfaces (10 Base* ABCs), backend registry |
-| `dataenginex.cli` | — | `dex` CLI (validate, version, init, serve) |
-| `dataenginex.api` | — | FastAPI app, auth (JWT), health, rate limiting |
-| `dataenginex.data` | — | Connectors, schema registry, profiler |
-| `dataenginex.ml` | — | Training, model registry, serving, drift detection |
-| `dataenginex.middleware` | — | Structured logging, Prometheus metrics, tracing |
-| `dataenginex.lakehouse` | optional `[cloud]` | Storage backends (local, S3, GCS), catalog |
-| `dataenginex.warehouse` | — | SQL/Spark transforms, lineage |
-| `dataenginex.plugins` | — | Plugin system (entry-point discovery) |
+| Module | Description |
+|--------|-------------|
+| `dataenginex.engine` | `DexEngine` — single entry point; loads config, inits store, wires all backends |
+| `dataenginex.store` | `DexStore` — DuckDB-backed persistence (`.dex/store.duckdb`) |
+| `dataenginex.config` | `dex.yaml` schema, loader, env var resolution, layering |
+| `dataenginex.core` | Exceptions, `Base*` ABCs, `BackendRegistry` |
+| `dataenginex.cli` | `dex` CLI (`validate`, `version`, `init`) |
+| `dataenginex.api` | HTTP helpers: error types, response models (no server bundled) |
+| `dataenginex.data` | Connectors, pipeline runner, schema registry, profiler |
+| `dataenginex.ml` | Classical ML: training, model registry, serving, drift detection |
+| `dataenginex.ai` | LLM providers, agents, RAG, vectorstore, memory, observability |
+| `dataenginex.orchestration` | `DriftScheduler`, background scheduling |
+| `dataenginex.middleware` | structlog config, Prometheus metrics |
+| `dataenginex.lakehouse` | Storage backends (local, S3, GCS), catalog, partitioning |
+| `dataenginex.warehouse` | SQL transforms, lineage tracking |
+| `dataenginex.plugins` | Entry-point plugin discovery |
 
 ## Quick Usage
 
 ```python
-# Config system
+from pathlib import Path
+from dataenginex.engine import DexEngine
+
+# Load config and initialize all backends
+engine = DexEngine(Path("dex.yaml"))
+
+# Data
+engine.run_pipeline("clean_users")
+sources = list(engine.config.data.sources.keys())
+
+# ML
+models = engine.model_registry.list_models()
+result = engine.model_registry.predict("churn_model", features)
+
+# AI
+response = engine.agents["assistant"].chat("summarize the latest run")
+
+# Persistence (DuckDB)
+runs = engine.store.list_pipeline_runs(limit=10)
+```
+
+```python
+# Config system only
 from dataenginex.config import load_config
 cfg = load_config(Path("dex.yaml"))
 
@@ -105,15 +123,12 @@ cfg = load_config(Path("dex.yaml"))
 from dataenginex.core.interfaces import BaseConnector
 from dataenginex.core.registry import BackendRegistry
 
-# Exceptions
-from dataenginex.core.exceptions import DataEngineXError, BackendNotInstalledError
-
 # ML
-from dataenginex.ml import ModelRegistry
+from dataenginex.ml import ModelRegistry, SklearnTrainer
 
-# CLI
-# dex validate dex.yaml
-# dex version
+# AI
+from dataenginex.ai.llm import get_llm_provider
+from dataenginex.ai.vectorstore import VectorStoreBackend
 ```
 
 ## Source and Docs

dataenginex-1.1.2/README.md

@@ -0,0 +1,161 @@
+# DEX — Data + ML + AI Framework
+
+[![CI](https://github.com/TheDataEngineX/DEX/actions/workflows/ci.yml/badge.svg?branch=dev)](https://github.com/TheDataEngineX/DEX/actions/workflows/ci.yml)
+[![PyPI](https://img.shields.io/pypi/v/dataenginex)](https://pypi.org/project/dataenginex/)
+[![Python 3.13+](https://img.shields.io/badge/python-3.13+-blue.svg)](https://www.python.org/downloads/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](LICENSE)
+
+Unified Data + ML + AI **library**. One `dex.yaml` defines your entire project — from data ingestion through ML training to AI agents. Self-hosted, config-driven, production-ready.
+
+`dataenginex` is a **pure Python library**. It has no HTTP server. Your application owns the server layer.
+
+______________________________________________________________________
+
+## Quick Start
+
+```bash
+pip install dataenginex
+```
+
+```python
+from dataenginex.engine import DexEngine
+
+engine = DexEngine("dex.yaml")          # loads config, inits DuckDB store
+engine.run_pipeline("clean_users")      # execute a pipeline
+models = engine.model_registry.list_models()
+```
+
+```bash
+# Development
+git clone https://github.com/TheDataEngineX/DEX && cd DEX
+uv run poe check-all          # lint + typecheck + tests
+dex validate dex.yaml         # validate a config file
+dex version                   # show version + environment
+```
+
+______________________________________________________________________
+
+## What It Does
+
+```
+dex.yaml
+  ├── data:           CSV/Parquet/DuckDB → transforms → quality checks
+  ├── ml:             Experiment tracking → training → serving → drift detection
+  ├── ai:             LLM providers → retrieval (BM25/dense/hybrid) → agents
+  └── observability:  structlog + Prometheus metrics
+```
+
+**Opinionated defaults, swappable backends.** Everything works out of the box with
+built-in implementations. Swap any layer for industry tools via optional extras:
+
+```bash
+pip install "dataenginex[cloud]"          # S3 + GCS storage backends
+pip install "dataenginex[observability]"  # Langfuse LLM tracing
+```
+
+______________________________________________________________________
+
+## Project Structure
+
+```
+dataenginex/
+├── src/dataenginex/
+│   ├── cli/                # dex CLI (validate, version, init)
+│   ├── config/             # dex.yaml schema, loader, env var resolution
+│   ├── core/               # Exceptions, interfaces (Base* ABCs), registry
+│   ├── engine.py           # DexEngine — single entry point for applications
+│   ├── store.py            # DexStore — DuckDB-backed persistence (.dex/store.duckdb)
+│   ├── api/                # HTTP helpers: error types, response models (no server)
+│   ├── data/               # Connectors, schema registry, profiler, pipeline runner
+│   ├── ml/                 # Classical ML: training, registry, serving, drift
+│   ├── ai/                 # LLM providers, agents, RAG, vectorstore, observability
+│   ├── orchestration/      # DriftScheduler, background workers
+│   ├── middleware/         # structlog config, Prometheus metrics (library use)
+│   ├── lakehouse/          # Catalog, partitioning, storage backends
+│   ├── warehouse/          # SQL transforms, lineage
+│   └── plugins/            # Plugin system (entry-point discovery)
+│
+├── examples/               # Runnable examples + dex.yaml templates
+├── tests/                  # Unit + integration tests
+├── docs/                   # MkDocs documentation
+└── pyproject.toml          # Package config (version source of truth)
+```
+
+______________________________________________________________________
+
+## Architecture
+
+```
+dex.yaml → DexEngine.__init__
+               │
+               ├── config/   load + validate → DexConfig
+               ├── store/    DexStore (.dex/store.duckdb)
+               ├── data/     register sources + pipelines
+               ├── ml/       model registry + serving
+               └── ai/       LLM providers + agents
+```
+
+**Backend Registry Pattern:** Every subsystem has a `Base*` ABC + `BackendRegistry`.
+Built-in backends work out of the box. Extras implement the same interface.
+
+**Tech Stack:**
+
+| Layer | Built-in | Optional Extra |
+|-------|----------|----------------|
+| Data Engine | DuckDB | PySpark |
+| Orchestration | croniter scheduler | Dagster |
+| ML Tracking | JSON-based tracker | MLflow |
+| Model Serving | Built-in predictor | — |
+| LLM | Ollama / LiteLLM / vLLM | Any OpenAI-compatible |
+| Vector Store | DuckDB VSS | Qdrant |
+| Retrieval | BM25 + Dense + Hybrid | — |
+| Persistence | DuckDB (`.dex/store.duckdb`) | — |
+| Logging | structlog | — |
+
+______________________________________________________________________
+
+## Development
+
+See [docs/development.md](docs/development.md) for full setup.
+
+```bash
+uv run poe check-all         # lint + typecheck + tests
+uv run poe lint-fix          # auto-fix lint issues
+uv run poe test-cov          # tests + coverage report
+```
+
+______________________________________________________________________
+
+## Documentation
+
+| Guide | Description |
+|-------|-------------|
+| [Quickstart](docs/quickstart.md) | Get running in 5 minutes |
+| [Architecture](docs/architecture.md) | System design and patterns |
+| [Development](docs/development.md) | Local setup and workflow |
+| [API Reference](docs/api-reference/index.md) | Auto-generated module docs |
+
+> Docs: [docs.thedataenginex.org](https://docs.thedataenginex.org)
+
+______________________________________________________________________
+
+## The DEX Ecosystem
+
+```
+TheDataEngineX/
+├── dataenginex    — Core library (this repo, PyPI)
+├── dex-studio     — Web UI (FastAPI + Jinja2) — single pane of glass
+└── infradex       — Terraform + Helm + K3s deployment
+```
+
+dex-studio imports `dataenginex` directly — no HTTP server required.
+
+______________________________________________________________________
+
+## License
+
+MIT License. See [LICENSE](LICENSE).
+
+______________________________________________________________________
+
+**Version**: [![PyPI](https://img.shields.io/pypi/v/dataenginex)](https://pypi.org/project/dataenginex/) | **License**: MIT | **Python**: 3.13+

dataenginex-1.1.2/docs/api-reference/api.md

@@ -0,0 +1,63 @@
+# dataenginex.api
+
+HTTP helpers — error types, response models, and shared utilities.
+
+`dataenginex` does **not** bundle a FastAPI server. This module provides the building blocks
+for applications (like DEX Studio or your own FastAPI app) that want to expose DEX functionality
+over HTTP.
+
+## Error Types
+
+```python
+from dataenginex.api.errors import (
+    NotFoundError,
+    ValidationError,
+    ConflictError,
+    ServiceUnavailableError,
+)
+```
+
+Standard Pydantic response models for HTTP error responses, usable in any FastAPI app:
+
+```python
+from fastapi import FastAPI
+from dataenginex.api.errors import NotFoundError
+
+app = FastAPI()
+
+@app.get("/pipelines/{name}")
+def get_pipeline(name: str):
+    pipeline = engine.config.data.pipelines.get(name)
+    if pipeline is None:
+        raise NotFoundError(detail=f"Pipeline '{name}' not found")
+    return pipeline
+```
+
+## Building an HTTP Layer
+
+To expose DEX functionality over HTTP, create a FastAPI app in your application
+and call `DexEngine` directly:
+
+```python
+from fastapi import FastAPI
+from dataenginex.engine import DexEngine
+
+engine = DexEngine("dex.yaml")
+app = FastAPI()
+
+@app.get("/health")
+def health():
+    return engine.health()
+
+@app.get("/pipelines")
+def list_pipelines():
+    return list(engine.config.data.pipelines.keys())
+
+@app.post("/pipelines/{name}/run")
+def run_pipeline(name: str):
+    return engine.run_pipeline(name)
+```
+
+See `examples/02_api_quickstart.py` for a minimal working example.
+
+::: dataenginex.api

dataenginex-1.1.2/docs/api-reference/dashboard.md

@@ -0,0 +1,8 @@
+---
+title: dashboard (removed)
+description: The Streamlit dashboard module was removed in v1.0
+---
+
+The `dataenginex.dashboard` Streamlit module was removed in **v1.0**.
+
+Use [DEX Studio](https://github.com/TheDataEngineX/dex-studio) instead — a full-featured Reflex web UI for data pipelines, ML experiments, and AI agents.

dataenginex-1.1.2/docs/api-reference/ml.md

@@ -0,0 +1,45 @@
+# dataenginex.ml
+
+Classical ML — training, model registry, drift detection, and model serving.
+
+LLM providers, vector stores, agents, and RAG live in `dataenginex.ai`.
+The drift scheduler lives in `dataenginex.orchestration`.
+
+## Module Split
+
+| Concern | Module |
+|---------|--------|
+| Training, registry, serving, drift | `dataenginex.ml` |
+| LLM providers, chat, embeddings | `dataenginex.ai.llm` |
+| Vector stores | `dataenginex.ai.vectorstore` |
+| Background drift scheduling | `dataenginex.orchestration.scheduler` |
+
+## Quick Usage
+
+```python
+from dataenginex.ml import (
+    SklearnTrainer, TrainingResult,
+    ModelRegistry, ModelArtifact, ModelStage,
+    DriftDetector, DriftReport,
+    ModelServer, PredictionRequest, PredictionResponse,
+)
+
+# Train
+trainer = SklearnTrainer(experiment_name="churn")
+result: TrainingResult = trainer.train(X_train, y_train)
+
+# Register
+registry = ModelRegistry()
+registry.register(result.model, name="churn_v1", stage=ModelStage.STAGING)
+
+# Drift
+detector = DriftDetector(reference=X_train)
+report: DriftReport = detector.detect(X_new)
+
+# Serve
+server = ModelServer()
+server.load("churn_v1", stage=ModelStage.PRODUCTION)
+resp = server.predict(PredictionRequest(features={"age": 35}))
+```
+
+::: dataenginex.ml