dataenginex 0.4.2__tar.gz → 0.4.4__tar.gz

{dataenginex-0.4.2 → dataenginex-0.4.4}/.claude/commands/new-feature.md

@@ -1,4 +1,4 @@
-# New Feature (DEX)
+# New Feature (DataEngineX)
 
 Scaffold a new feature for the `dataenginex` core framework. Ask for the feature name and description if not provided as $ARGUMENTS.
 

{dataenginex-0.4.2 → dataenginex-0.4.4}/.claude/commands/validate.md

@@ -1,6 +1,6 @@
-# Validate (DEX)
+# Validate (DataEngineX)
 
-Run the full DEX validation pipeline in order. Stop and report on first failure.
+Run the full DataEngineX validation pipeline in order. Stop and report on first failure.
 
 1. **Lint**
 
@@ -25,10 +25,11 @@ Run the full DEX validation pipeline in order. Stop and report on first failure.
    ```bash
    uv run python examples/02_api_quickstart.py &
    sleep 2
-   curl -sf http://localhost:17000/health   | python -m json.tool
-   curl -sf http://localhost:17000/         | python -m json.tool
-   curl -sf http://localhost:17000/metrics  | head -20
-   curl -sf -X POST http://localhost:17000/echo -H "Content-Type: application/json" -d '{"message":"test"}' | python -m json.tool
+   # dataenginex has no built-in HTTP server; these endpoints apply to dex-studio
+   curl -sf http://localhost:7860/health   | python -m json.tool
+   curl -sf http://localhost:7860/         | python -m json.tool
+   curl -sf http://localhost:7860/metrics  | head -20
+   curl -sf -X POST http://localhost:7860/echo -H "Content-Type: application/json" -d '{"message":"test"}' | python -m json.tool
    ```
 
    Check response **bodies** — not just status codes. Kill the server after.

{dataenginex-0.4.2 → dataenginex-0.4.4}/.github/dependabot.yml

@@ -4,9 +4,7 @@ updates:
     directory: "/"
     schedule:
       interval: weekly
-    target-branch: dev
   - package-ecosystem: github-actions
     directory: "/"
     schedule:
       interval: weekly
-    target-branch: dev

{dataenginex-0.4.2 → dataenginex-0.4.4}/.github/release-pr-template.md

@@ -1,6 +1,6 @@
 # Release PR Template
 
-Use this template whenever opening a release PR (`dev → main`). Reference `release-notes.md` and verify QA steps before merging.
+Use this template whenever opening a release PR to `main`. Reference `release-notes.md` and verify QA steps before merging.
 
 ## Summary
 

dataenginex-0.4.4/.github/workflows/auto-pr.yml

@@ -0,0 +1,16 @@
+name: Auto PR
+
+on:
+  push:
+    branches:
+      - '*'
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  auto-pr:
+    if: github.ref_name != 'main'
+    uses: TheDataEngineX/.github/.github/workflows/reusable-auto-pr-to-main.yml@main
+    secrets: inherit

{dataenginex-0.4.2 → dataenginex-0.4.4}/.github/workflows/ci.yml

@@ -2,9 +2,9 @@ name: CI
 
 on:
   push:
-    branches: [main, dev]
+    branches: [main]
   pull_request:
-    branches: [main, dev]
+    branches: [main]
   workflow_dispatch:
 
   schedule:
@@ -19,7 +19,7 @@ jobs:
     name: Code Quality
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - uses: astral-sh/setup-uv@v8.2.0
         with:
           version: "latest"
@@ -27,14 +27,18 @@ jobs:
       - run: uv sync --group ml
         env:
           UV_PROJECT_ENVIRONMENT: .venv
+      - uses: reviewdog/action-actionlint@v1
+
       - run: uv run poe quality
 
+      - run: uv build
+
   test:
     name: Tests (Python 3.13)
     runs-on: ubuntu-latest
     needs: quality
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - uses: astral-sh/setup-uv@v8.2.0
         with:
           version: "latest"
@@ -58,7 +62,7 @@ jobs:
       matrix:
         python-version: ["3.11", "3.12"]
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - uses: astral-sh/setup-uv@v8.2.0
         with:
           version: "latest"

{dataenginex-0.4.2 → dataenginex-0.4.4}/.github/workflows/release.yml

@@ -23,7 +23,7 @@ jobs:
     permissions:
       contents: read
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
         with:
           ref: ${{ inputs.ref || github.ref }}
       - uses: astral-sh/setup-uv@v8.2.0
@@ -58,7 +58,7 @@ jobs:
     permissions:
       contents: write
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
         with:
           ref: ${{ inputs.ref || github.ref }}
       - uses: astral-sh/setup-uv@v8.2.0
@@ -80,6 +80,6 @@ jobs:
           RELEASE_TAG: ${{ inputs.tag || github.ref_name }}
         run: |
           gh release create "$RELEASE_TAG" \
-            --title "DEX $RELEASE_TAG" \
+            --title "dataenginex $RELEASE_TAG" \
             --generate-notes \
             "sbom-dex-${{ steps.version.outputs.version }}.json"

{dataenginex-0.4.2 → dataenginex-0.4.4}/.github/workflows/security.yml

@@ -2,9 +2,9 @@ name: Security Scans
 
 on:
   push:
-    branches: [main, dev]
+    branches: [main]
   pull_request:
-    branches: [main, dev]
+    branches: [main]
   workflow_dispatch:
 
 permissions:
@@ -13,5 +13,5 @@ permissions:
 
 jobs:
   security:
-    uses: TheDataEngineX/.github/.github/workflows/security.yml@main
+    uses: TheDataEngineX/.github/.github/workflows/reusable-security.yml@main
     secrets: inherit

{dataenginex-0.4.2 → dataenginex-0.4.4}/CLAUDE.md

@@ -1,4 +1,4 @@
-# CLAUDE.md — DEX (dataenginex)
+# CLAUDE.md — DataEngineX (dataenginex)
 
 Brief answers only. No explanations unless asked.
 Goal is to save Claude code tokens for lower cost without losing quality.
@@ -7,7 +7,7 @@ Goal is to save Claude code tokens for lower cost without losing quality.
 
 ## Project Overview
 
-**DEX** — unified Data + ML + AI library. Config-driven, self-hosted, local-first. Pure Python — no bundled HTTP server.
+**DataEngineX** — unified Data + ML + AI library. Config-driven, self-hosted, local-first. Pure Python — no bundled HTTP server.
 
 | Package | Location | Purpose |
 |---------|----------|---------|
@@ -41,7 +41,7 @@ dex validate dex.yaml     # Validate config file
 dex version               # Show version + environment
 
 # Dev
-uv run poe dev            # Dev server (uvicorn reload, port 17000) — for examples/API testing only
+uv run poe dev            # Dev server (uvicorn reload, port 17000) — for examples/API testing only (dataenginex has no built-in HTTP server; see dex-studio for web UI)
 uv run poe docker-up      # Docker compose up
 uv run poe docker-down    # Docker compose down
 
@@ -63,7 +63,7 @@ pip install 'litellm>=1.83.3' --no-deps # LLM routing (separate: pins python-dot
 
 ______________________________________________________________________
 
-## Mandatory Validation
+## Validation
 
-Run `/validate` after ANY code change. Steps 4-5 (config validation + server smoke test) are NON-NEGOTIABLE.
-Tests passing ≠ app working.
+After any code change run: `uv run poe check-all` (lint + typecheck + test).
+Tests passing ≠ app working — run `dex validate dex.yaml` to verify config.

{dataenginex-0.4.2 → dataenginex-0.4.4}/CONTRIBUTING.md

@@ -1,18 +1,18 @@
-# Contributing to DEX
+# Contributing to DataEngineX
 
 Thank you for your interest in contributing to DataEngineX!
 
-**→ Full contributor guide: [docs/CONTRIBUTING.md](docs/CONTRIBUTING.md)**
+**→ Full contributor guide: [docs/contributing.md](docs/contributing.md)**
 
 Quick essentials:
 
-- Development setup: [docs/DEVELOPMENT.md](docs/DEVELOPMENT.md)
-- Governance & legal: See [docs/CONTRIBUTING.md](docs/CONTRIBUTING.md)
+- Development setup: [docs/development.md](docs/development.md)
+- Governance & legal: See [docs/contributing.md](docs/contributing.md)
 - Code of Conduct: [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md)
 
 ## Quick Start
 
-1. Fork and create a feature branch from `dev`
+1. Fork and create a feature branch from `main`
 1. Run local checks: `uv run poe check-all`
 1. Open a PR using `.github/PULL_REQUEST_TEMPLATE.md`
 

{dataenginex-0.4.2 → dataenginex-0.4.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dataenginex
-Version: 0.4.2
+Version: 0.4.4
 Summary: DataEngineX — open-source, self-hosted, local-first Data + ML + AI workbench library
 Author-email: Jay <jayapal.myaka99@gmail.com>
 License: MIT License
@@ -30,7 +30,6 @@ Requires-Dist: click>=8.3.3
 Requires-Dist: croniter>=6.2.2
 Requires-Dist: duckdb>=1.5.2
 Requires-Dist: httpx>=0.28.1
-Requires-Dist: msgpack>=1.2.1
 Requires-Dist: orjson>=3.11.0
 Requires-Dist: prometheus-client>=0.25.0
 Requires-Dist: pyarrow>=23.0.1
@@ -38,7 +37,6 @@ Requires-Dist: pydantic-settings>=2.14.2
 Requires-Dist: pydantic>=2.13.4
 Requires-Dist: pyyaml>=6.0.3
 Requires-Dist: structlog>=25.5.0
-Requires-Dist: zstandard>=0.25.0
 Provides-Extra: cloud
 Requires-Dist: boto3>=1.43.7; extra == 'cloud'
 Requires-Dist: google-cloud-bigquery>=3.41.0; extra == 'cloud'

{dataenginex-0.4.2 → dataenginex-0.4.4}/README.md

@@ -1,13 +1,11 @@
 # dataenginex
 
-[![CI](https://github.com/TheDataEngineX/dataenginex/actions/workflows/ci.yml/badge.svg?branch=dev)](https://github.com/TheDataEngineX/dataenginex/actions/workflows/ci.yml)
+[![CI](https://github.com/TheDataEngineX/dataenginex/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/TheDataEngineX/dataenginex/actions/workflows/ci.yml)
 [![PyPI](https://img.shields.io/pypi/v/dataenginex)](https://pypi.org/project/dataenginex/)
 [![Python 3.13+](https://img.shields.io/badge/python-3.13+-blue.svg)](https://www.python.org/downloads/)
 [![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](LICENSE)
 
-The Python library that powers [DEX Studio](https://github.com/TheDataEngineX/dex-studio) — an open-source, self-hosted, local-first Data + ML + AI workbench for individuals and small teams. **Use the library directly when you want code; install DEX Studio when you want a UI.**
-
-> **Pre-1.0 status.** `0.4.0` is honest about it. See the [CHANGELOG scope-reset entry](CHANGELOG.md) for the rationale.
+The Python library that powers [DataEngineX Studio](https://github.com/TheDataEngineX/dex-studio) — an open-source, self-hosted, local-first Data + ML + AI workbench for individuals and small teams. **Use the library directly when you want code; install DataEngineX Studio when you want a UI.**
 
 ______________________________________________________________________
 
@@ -118,7 +116,7 @@ git clone https://github.com/TheDataEngineX/dex-studio && cd dex-studio
 docker compose up             # http://localhost:7860
 ```
 
-DEX Studio imports `dataenginex` directly — no separate API server, no HTTP hop.
+DataEngineX Studio imports `dataenginex` directly — no separate API server, no HTTP hop.
 
 ______________________________________________________________________
 

{dataenginex-0.4.2 → dataenginex-0.4.4}/docker-compose.test.yml

@@ -1,5 +1,5 @@
 # ---------------------------------------------------------------
-# DEX — Storage Emulators for Integration Tests
+# DataEngineX — Storage Emulators for Integration Tests
 #
 # Spins up LocalStack (S3) and fake-gcs-server (GCS) for local
 # integration testing without real cloud accounts.

{dataenginex-0.4.2 → dataenginex-0.4.4}/docs/adr/0001-medallion-architecture.md

@@ -8,18 +8,18 @@
 
 ## Context
 
-DEX needs a scalable, standardized data architecture that works across all projects (CareerDEX, Weather, etc.). The data pipeline needs to handle:
+DataEngineX needs a scalable, standardized data architecture that works across all projects (CareerDEX, Weather, etc.). The data pipeline needs to handle:
 
 - Raw data ingestion from multiple sources
 - Data transformation and quality validation
 - Feature engineering for ML models
 - Analytics-ready data consumption
 
-Multiple projects will be built on the DEX platform, each with different data sources and transformations. We need a consistent, reusable pattern.
+Multiple projects will be built on the DataEngineX platform, each with different data sources and transformations. We need a consistent, reusable pattern.
 
 ## Decision
 
-Implement the **Medallion Architecture** (Bronze → Silver → Gold layers) as the standard data architecture for all DEX projects.
+Implement the **Medallion Architecture** (Bronze → Silver → Gold layers) as the standard data architecture for all DataEngineX projects.
 
 ### Architecture Layers
 

dataenginex-0.4.4/docs/api-reference/ai.md

@@ -0,0 +1,191 @@
+# dataenginex.ai
+
+LLM routing, agent runtimes, vector stores, memory, retrieval, observability, and workflow orchestration.
+
+## Quick import
+
+```python
+from dataenginex.ai import (
+    ModelRouter, BaseProvider,
+    LLMProvider, LLMResponse,
+    BuiltinAgentRuntime,
+    BuiltinRetriever,
+    VectorStore,
+    SandboxConfig,
+)
+```
+
+______________________________________________________________________
+
+## LLM Interface
+
+`dataenginex.ai.llm`
+
+Unified LLM request/response interface across providers. Handles streaming, tool calls, retries, and token counting.
+
+::: dataenginex.ai.llm
+
+**Key classes:** `LLMProvider`, `LLMResponse`, `LLMMessage`, `ToolCall`
+
+```python
+from dataenginex.ai.llm import LLMMessage
+
+response = provider.complete([
+    LLMMessage(role="user", content="Summarize this dataset."),
+])
+print(response.content)
+```
+
+______________________________________________________________________
+
+## Model Router
+
+`dataenginex.ai.routing.router`
+
+Routes LLM requests to the appropriate provider based on cost, latency, capability, and fallback rules.
+
+::: dataenginex.ai.routing.router
+
+**Key class:** `ModelRouter`
+
+```python
+from dataenginex.ai.routing.router import ModelRouter
+
+router = ModelRouter.from_config(engine.config)
+response = router.complete("Explain this error.", model_hint="fast")
+```
+
+### Providers
+
+`dataenginex.ai.routing.anthropic` · `dataenginex.ai.routing.openai` · `dataenginex.ai.routing.ollama` · `dataenginex.ai.routing.guarded`
+
+::: dataenginex.ai.routing.anthropic
+::: dataenginex.ai.routing.openai
+::: dataenginex.ai.routing.ollama
+::: dataenginex.ai.routing.guarded
+
+______________________________________________________________________
+
+## Agents
+
+`dataenginex.ai.agents.builtin`
+
+Built-in agent runtime — tool-use loop, memory injection, step tracing, and structured output parsing.
+
+::: dataenginex.ai.agents.builtin
+
+**Key class:** `BuiltinAgentRuntime`
+
+```python
+from dataenginex.ai.agents.builtin import BuiltinAgentRuntime
+
+agent = BuiltinAgentRuntime(router=router, tools=[search_tool, sql_tool])
+result = agent.run("Find the top 10 customers by revenue last quarter.")
+print(result.output)
+```
+
+______________________________________________________________________
+
+## Vector Store
+
+`dataenginex.ai.vectorstore`
+
+Embedding storage and similarity search. Defaults to in-process DuckDB VSS; swap for Qdrant via `dataenginex[qdrant]`.
+
+::: dataenginex.ai.vectorstore
+
+**Key class:** `VectorStore`
+
+```python
+from dataenginex.ai.vectorstore import VectorStore
+
+store = VectorStore(db_path=".dex/store.duckdb")
+store.upsert("doc-1", embedding=[0.1, 0.2, ...], metadata={"source": "wiki"})
+results = store.search(query_embedding, top_k=5)
+```
+
+______________________________________________________________________
+
+## Memory
+
+`dataenginex.ai.memory.base` — abstract memory interface
+
+::: dataenginex.ai.memory.base
+
+`dataenginex.ai.memory.episodic` — short-term conversation memory scoped to a single agent session
+
+::: dataenginex.ai.memory.episodic
+
+`dataenginex.ai.memory.long_term` — persistent memory backed by the vector store, survives across sessions
+
+::: dataenginex.ai.memory.long_term
+
+______________________________________________________________________
+
+## Retrieval
+
+`dataenginex.ai.retrieval.builtin` — RAG retriever: embeds query, searches vector store, returns ranked chunks
+
+::: dataenginex.ai.retrieval.builtin
+
+`dataenginex.ai.retrieval.graph` — graph-based retrieval for structured knowledge graphs
+
+::: dataenginex.ai.retrieval.graph
+
+______________________________________________________________________
+
+## Runtime
+
+`dataenginex.ai.runtime.executor` — async execution engine with concurrency, timeout, and step-level error handling
+
+::: dataenginex.ai.runtime.executor
+
+`dataenginex.ai.runtime.checkpoint` — saves and restores agent run state for long-running or resumable workflows
+
+::: dataenginex.ai.runtime.checkpoint
+
+`dataenginex.ai.runtime.sandbox` — isolated code execution sandbox for agent-generated Python with configurable resource limits
+
+::: dataenginex.ai.runtime.sandbox
+
+______________________________________________________________________
+
+## Tools
+
+`dataenginex.ai.tools.builtin`
+
+Built-in agent tools: `sql_query`, `web_search`, `file_read`, `python_exec`, `vector_search`.
+
+::: dataenginex.ai.tools.builtin
+
+______________________________________________________________________
+
+## Workflows
+
+`dataenginex.ai.workflows.dag` — multi-step agent workflows as DAGs; steps branch, merge, and pass structured outputs
+
+::: dataenginex.ai.workflows.dag
+
+`dataenginex.ai.workflows.conditions` — conditional branching logic for DAG workflows
+
+::: dataenginex.ai.workflows.conditions
+
+`dataenginex.ai.workflows.human_loop` — pause a workflow at a step requiring human review or approval
+
+::: dataenginex.ai.workflows.human_loop
+
+______________________________________________________________________
+
+## Observability
+
+`dataenginex.ai.observability.audit` — logs every LLM request/response, tool call, and agent step for compliance
+
+::: dataenginex.ai.observability.audit
+
+`dataenginex.ai.observability.cost` — tracks token usage and estimated cost per provider, model, and agent run
+
+::: dataenginex.ai.observability.cost
+
+`dataenginex.ai.observability.metrics` — Prometheus metrics for LLM latency, token throughput, error rate
+
+::: dataenginex.ai.observability.metrics

dataenginex-0.4.4/docs/api-reference/config.md

@@ -0,0 +1,84 @@
+# dataenginex.config
+
+Config schema, loader, settings, and defaults. The `dex.yaml` file is parsed into a typed `DexConfig` object that drives the engine.
+
+## Quick import
+
+```python
+from dataenginex.config import DexConfig, load_config, DexSettings
+```
+
+______________________________________________________________________
+
+## Schema
+
+`dataenginex.config.schema`
+
+Pydantic models for `dex.yaml`. Covers pipelines, connectors, transforms, ML, AI, secops, and lakehouse config blocks.
+
+::: dataenginex.config.schema
+
+**Key model:** `DexConfig`
+
+```python
+from dataenginex.config.schema import DexConfig
+
+config = DexConfig.model_validate({
+    "pipelines": {
+        "ingest_events": {
+            "source": {"type": "csv", "path": "data/raw/events.csv"},
+            "destination": {"type": "parquet", "path": "data/bronze/events"},
+        }
+    }
+})
+```
+
+______________________________________________________________________
+
+## Loader
+
+`dataenginex.config.loader`
+
+Loads and validates `dex.yaml` (or a custom path) with environment variable interpolation and secret resolution.
+
+::: dataenginex.config.loader
+
+**Key function:** `load_config`
+
+```python
+from dataenginex.config.loader import load_config
+
+config = load_config("dex.yaml")
+```
+
+______________________________________________________________________
+
+## Settings
+
+`dataenginex.config.settings`
+
+Runtime settings sourced from environment variables (`.env` or system env). Covers API keys, storage paths, log level, and feature flags.
+
+::: dataenginex.config.settings
+
+**Key class:** `DexSettings`
+
+```python
+from dataenginex.config.settings import DexSettings
+
+settings = DexSettings()
+print(settings.log_level)       # "INFO"
+print(settings.dex_store_path)  # ".dex/store.duckdb"
+```
+
+See `.env.template` in the project root for all available settings.
+
+______________________________________________________________________
+
+## Defaults
+
+`dataenginex.config.defaults`
+
+Default values applied when config keys are omitted.
+
+::: dataenginex.config.defaults