dataenginex 1.1.0__tar.gz → 1.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dataenginex-1.1.1/.github/workflows/auto-pr.yml +23 -0
- dataenginex-1.1.1/.github/workflows/release.yml +69 -0
- dataenginex-1.1.1/.release-please-manifest.json +3 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/CHANGELOG.md +7 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/PKG-INFO +1 -2
- {dataenginex-1.1.0 → dataenginex-1.1.1}/docs/release-notes.md +25 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/poe_tasks.toml +0 -13
- {dataenginex-1.1.0 → dataenginex-1.1.1}/pyproject.toml +1 -2
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/__init__.py +1 -1
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/cli/main.py +29 -25
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/cli/run.py +14 -17
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/cli/train.py +21 -31
- {dataenginex-1.1.0 → dataenginex-1.1.1}/uv.lock +1 -3
- dataenginex-1.1.0/.github/workflows/auto-pr-dev-to-main.yml +0 -15
- dataenginex-1.1.0/.github/workflows/auto-pr-to-dev.yml +0 -16
- dataenginex-1.1.0/.github/workflows/claude.yml +0 -23
- dataenginex-1.1.0/.github/workflows/pypi-publish.yml +0 -290
- dataenginex-1.1.0/.github/workflows/release-dex.yml +0 -150
- dataenginex-1.1.0/.github/workflows/release-please.yml +0 -16
- dataenginex-1.1.0/.release-please-manifest.json +0 -3
- {dataenginex-1.1.0 → dataenginex-1.1.1}/.claude/commands/new-feature.md +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/.claude/commands/validate.md +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/.claude/settings.json +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/.env.template +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/.github/ISSUE_TEMPLATE/bug_report.yml +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/.github/ISSUE_TEMPLATE/config.yml +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/.github/ISSUE_TEMPLATE/feature_request.yml +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/.github/PULL_REQUEST_TEMPLATE.md +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/.github/dependabot.yml +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/.github/labels.yml +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/.github/release-pr-template.md +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/.github/workflows/ci.yml +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/.github/workflows/docker-build-push.yml +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/.github/workflows/docs-notify.yml +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/.github/workflows/enforce-dev-to-main.yml +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/.github/workflows/label-sync.yml +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/.github/workflows/security.yml +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/.gitignore +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/.gitleaks.toml +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/.pre-commit-config.yaml +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/.python-version +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/CLAUDE.md +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/CODEOWNERS +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/CODE_OF_CONDUCT.md +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/CONTRIBUTING.md +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/Dockerfile +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/LICENSE +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/README.md +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/SECURITY.md +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/docker-compose.test.yml +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/docs/adr/0000-template.md +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/docs/adr/0001-medallion-architecture.md +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/docs/api-reference/api.md +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/docs/api-reference/core.md +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/docs/api-reference/dashboard.md +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/docs/api-reference/data.md +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/docs/api-reference/index.md +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/docs/api-reference/lakehouse.md +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/docs/api-reference/middleware.md +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/docs/api-reference/ml.md +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/docs/api-reference/plugins.md +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/docs/api-reference/warehouse.md +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/docs/architecture.md +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/docs/ci-cd.md +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/docs/contributing.md +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/docs/development.md +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/docs/index.md +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/docs/observability.md +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/docs/quickstart.md +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/docs/sdlc.md +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/docs/security-scanning.md +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/examples/01_hello_pipeline.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/examples/02_api_quickstart.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/examples/03_quality_gate.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/examples/04_ml_training.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/examples/05_rag_demo.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/examples/06_llm_quickstart.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/examples/07_api_ingestion.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/examples/08_spark_ml.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/examples/09_feature_engineering.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/examples/10_model_analysis.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/examples/GUIDE.md +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/examples/dashboard/dashboard_config.yaml +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/examples/dashboard/run_dashboard.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/examples/data/events.csv +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/examples/data/users.csv +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/examples/dex.yaml +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/examples/ecommerce/data/customers.csv +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/examples/ecommerce/data/orders.csv +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/examples/ecommerce/data/products.csv +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/examples/ecommerce/dex.yaml +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/examples/ecommerce/run_all.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/examples/movies.csv +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/release-please-config.json +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/sonar-project.properties +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/README.md +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/RELEASE_NOTES.md +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/ai/__init__.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/ai/agents/__init__.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/ai/agents/builtin.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/ai/memory/__init__.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/ai/memory/base.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/ai/memory/episodic.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/ai/memory/long_term.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/ai/observability/__init__.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/ai/observability/audit.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/ai/observability/cost.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/ai/observability/langfuse.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/ai/observability/metrics.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/ai/retrieval/__init__.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/ai/retrieval/builtin.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/ai/retrieval/graph.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/ai/routing/__init__.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/ai/routing/anthropic.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/ai/routing/huggingface.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/ai/routing/ollama.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/ai/routing/openai.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/ai/routing/router.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/ai/runtime/__init__.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/ai/runtime/checkpoint.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/ai/runtime/executor.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/ai/runtime/sandbox.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/ai/tools/__init__.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/ai/tools/builtin.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/ai/workflows/__init__.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/ai/workflows/conditions.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/ai/workflows/dag.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/ai/workflows/human_loop.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/api/__init__.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/api/auth.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/api/errors.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/api/factory.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/api/health.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/api/jwks.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/api/ldap_sync.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/api/pagination.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/api/rate_limit.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/api/rbac.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/api/routers/__init__.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/api/routers/ai.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/api/routers/data.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/api/routers/health.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/api/routers/ml.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/api/routers/pipelines.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/api/routers/root.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/api/routers/system.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/api/schemas.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/api/scim.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/cli/__init__.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/cli/serve.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/config/__init__.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/config/defaults.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/config/loader.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/config/schema.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/core/__init__.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/core/exceptions.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/core/interfaces.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/core/medallion_architecture.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/core/quality.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/core/registry.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/core/schemas.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/core/validators.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/data/__init__.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/data/connectors/__init__.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/data/connectors/csv.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/data/connectors/duckdb.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/data/connectors/legacy.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/data/pipeline/__init__.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/data/pipeline/dag.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/data/pipeline/run_history.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/data/pipeline/runner.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/data/profiler.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/data/quality/__init__.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/data/quality/gates.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/data/quality/spark.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/data/registry.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/data/transforms/__init__.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/data/transforms/sql.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/lakehouse/__init__.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/lakehouse/catalog.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/lakehouse/partitioning.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/lakehouse/storage.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/middleware/__init__.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/middleware/domain_metrics.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/middleware/logging_config.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/middleware/metrics.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/middleware/metrics_middleware.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/middleware/request_logging.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/middleware/tracing.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/ml/__init__.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/ml/drift.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/ml/features/__init__.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/ml/features/builtin.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/ml/llm.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/ml/metrics.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/ml/mlflow_registry.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/ml/registry.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/ml/scheduler.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/ml/serving.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/ml/serving_engine/__init__.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/ml/serving_engine/builtin.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/ml/tracking/__init__.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/ml/tracking/builtin.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/ml/training.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/ml/vectorstore.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/orchestration/__init__.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/orchestration/builtin.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/plugins/__init__.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/plugins/registry.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/py.typed +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/secops/__init__.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/secops/audit.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/secops/gate.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/secops/masking.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/secops/pii.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/warehouse/__init__.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/warehouse/lineage.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/warehouse/transforms.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/src/dataenginex/worker.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/__init__.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/conformance/__init__.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/conformance/test_connector.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/conformance/test_feature_store.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/conformance/test_tracker.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/conformance/test_transform.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/conftest.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/fixtures/__init__.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/fixtures/sample_data.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/fixtures/sample_jobs.csv +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/fixtures/sample_jobs.json +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/integration/__init__.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/integration/test_ai_integration.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/integration/test_api_middleware_integration.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/integration/test_cli_run.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/integration/test_config_cli.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/integration/test_full_app.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/integration/test_lineage_integration.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/integration/test_ml_integration.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/integration/test_pipeline_e2e.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/integration/test_secops_integration.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/integration/test_storage_real.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/load/__init__.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/__init__.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_agent_runtime.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_ai_modules.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_ai_router.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_ai_router_extended.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_api_auth.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_api_factory.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_api_jwks.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_api_pagination.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_api_rate_limit.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_api_rbac.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_api_schemas.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_api_scim.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_api_validators.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_builtin_agent.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_builtin_feature_store.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_builtin_serving.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_builtin_tracker.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_cli_train.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_config_loader.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_config_schema.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_core_exceptions.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_core_interfaces.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_core_registry.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_csv_connector.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_data.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_data_router.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_domain_metrics.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_domain_metrics_wiring.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_drift_scheduler.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_duckdb_connector.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_errors.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_health.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_lakehouse.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_llm.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_llm_litellm_vllm.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_logging.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_medallion.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_metrics.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_middleware.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_ml.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_ml_router.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_observability_langfuse.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_pipeline_dag.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_pipeline_router_extended.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_pipeline_runner.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_plugins.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_quality_gates.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_quality_spark.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_retriever.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_retriever_graph.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_run_history.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_scheduler.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_secops.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_security_extended.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_spark_fixtures.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_sql_transforms.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_storage_abstraction.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_system_router.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_tracing.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_vectorstore.py +0 -0
- {dataenginex-1.1.0 → dataenginex-1.1.1}/tests/unit/test_warehouse.py +0 -0
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
name: Auto PR
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches:
|
|
6
|
+
- dev
|
|
7
|
+
- 'feature/**'
|
|
8
|
+
- 'fix/**'
|
|
9
|
+
|
|
10
|
+
permissions:
|
|
11
|
+
contents: write
|
|
12
|
+
pull-requests: write
|
|
13
|
+
|
|
14
|
+
jobs:
|
|
15
|
+
feature-to-dev:
|
|
16
|
+
if: startsWith(github.ref, 'refs/heads/feature/') || startsWith(github.ref, 'refs/heads/fix/')
|
|
17
|
+
uses: TheDataEngineX/.github/.github/workflows/auto-pr-to-dev.yml@main
|
|
18
|
+
secrets: inherit
|
|
19
|
+
|
|
20
|
+
dev-to-main:
|
|
21
|
+
if: github.ref == 'refs/heads/dev'
|
|
22
|
+
uses: TheDataEngineX/.github/.github/workflows/auto-pr-dev-to-main.yml@main
|
|
23
|
+
secrets: inherit
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
name: Release
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags:
|
|
6
|
+
- 'v[0-9]+.[0-9]+.[0-9]+'
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
build:
|
|
10
|
+
name: Build
|
|
11
|
+
runs-on: ubuntu-latest
|
|
12
|
+
permissions:
|
|
13
|
+
contents: read
|
|
14
|
+
steps:
|
|
15
|
+
- uses: actions/checkout@v6
|
|
16
|
+
- uses: astral-sh/setup-uv@v7
|
|
17
|
+
with:
|
|
18
|
+
version: "latest"
|
|
19
|
+
python-version: "3.13"
|
|
20
|
+
- run: uv build
|
|
21
|
+
- uses: actions/upload-artifact@v4
|
|
22
|
+
with:
|
|
23
|
+
name: dist
|
|
24
|
+
path: dist/
|
|
25
|
+
|
|
26
|
+
publish-pypi:
|
|
27
|
+
name: Publish to PyPI
|
|
28
|
+
needs: build
|
|
29
|
+
runs-on: ubuntu-latest
|
|
30
|
+
environment: ${{ vars.PYPI_ENVIRONMENT }}
|
|
31
|
+
permissions:
|
|
32
|
+
contents: read
|
|
33
|
+
id-token: write
|
|
34
|
+
steps:
|
|
35
|
+
- uses: actions/download-artifact@v4
|
|
36
|
+
with:
|
|
37
|
+
name: dist
|
|
38
|
+
path: dist/
|
|
39
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
40
|
+
|
|
41
|
+
github-release:
|
|
42
|
+
name: GitHub Release + SBOM
|
|
43
|
+
needs: build
|
|
44
|
+
runs-on: ubuntu-latest
|
|
45
|
+
permissions:
|
|
46
|
+
contents: write
|
|
47
|
+
steps:
|
|
48
|
+
- uses: actions/checkout@v6
|
|
49
|
+
- uses: astral-sh/setup-uv@v7
|
|
50
|
+
with:
|
|
51
|
+
version: "latest"
|
|
52
|
+
- name: Extract version
|
|
53
|
+
id: version
|
|
54
|
+
run: |
|
|
55
|
+
VERSION=$(grep -m 1 "^version = " pyproject.toml | sed 's/version = "//;s/".*//')
|
|
56
|
+
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
|
|
57
|
+
- name: Generate CycloneDX SBOM
|
|
58
|
+
run: |
|
|
59
|
+
uvx --from cyclonedx-bom cyclonedx-py environment \
|
|
60
|
+
-o sbom-dex-${{ steps.version.outputs.version }}.json \
|
|
61
|
+
--of json
|
|
62
|
+
- name: Create GitHub release + attach SBOM
|
|
63
|
+
env:
|
|
64
|
+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
65
|
+
run: |
|
|
66
|
+
gh release create ${{ github.ref_name }} \
|
|
67
|
+
--title "DEX ${{ github.ref_name }}" \
|
|
68
|
+
--generate-notes \
|
|
69
|
+
"sbom-dex-${{ steps.version.outputs.version }}.json"
|
|
@@ -5,6 +5,13 @@ All notable changes to `dataenginex` will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [1.1.1](https://github.com/TheDataEngineX/dex/compare/v1.1.0...v1.1.1) (2026-05-07)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
### Bug Fixes
|
|
12
|
+
|
|
13
|
+
* rich reflex compat ([#232](https://github.com/TheDataEngineX/dex/issues/232)) ([#233](https://github.com/TheDataEngineX/dex/issues/233)) ([7d33c05](https://github.com/TheDataEngineX/dex/commit/7d33c05d9d3b041567a5539930dabc30294a0d8d))
|
|
14
|
+
|
|
8
15
|
## [1.1.0](https://github.com/TheDataEngineX/dex/compare/v1.0.3...v1.1.0) (2026-05-06)
|
|
9
16
|
|
|
10
17
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dataenginex
|
|
3
|
-
Version: 1.1.
|
|
3
|
+
Version: 1.1.1
|
|
4
4
|
Summary: DataEngineX - Core framework for AI/ML/Data engineering projects
|
|
5
5
|
Author-email: Jay <jayapal.myaka99@gmail.com>
|
|
6
6
|
License: MIT License
|
|
@@ -45,7 +45,6 @@ Requires-Dist: python-dotenv>=1.2.2
|
|
|
45
45
|
Requires-Dist: pyyaml>=6.0.3
|
|
46
46
|
Requires-Dist: qdrant-client>=1.17.1
|
|
47
47
|
Requires-Dist: redis>=5.3.1
|
|
48
|
-
Requires-Dist: rich>=15.0.0
|
|
49
48
|
Requires-Dist: structlog>=25.5.0
|
|
50
49
|
Requires-Dist: uvicorn>=0.46.0
|
|
51
50
|
Provides-Extra: auth
|
|
@@ -1,5 +1,30 @@
|
|
|
1
1
|
# Release Notes
|
|
2
2
|
|
|
3
|
+
## v1.1.0 — 2026-05-06
|
|
4
|
+
|
|
5
|
+
### Highlights
|
|
6
|
+
|
|
7
|
+
- **Enterprise auth & RBAC** — SCIM provisioning, role-based access, enterprise SSO integration
|
|
8
|
+
- **LiteLLM / vLLM routing** — unified LLM gateway with cost tracking and load balancing
|
|
9
|
+
- **Langfuse observability** — LLM trace logging, evals, and prompt management
|
|
10
|
+
- **LightRAG integration** — graph-based retrieval for hybrid RAG pipelines
|
|
11
|
+
- **AI agent memory** — persistent agent memory subsystem with configurable backends
|
|
12
|
+
- **Domain metadata layer** — structured domain extraction and plugin system
|
|
13
|
+
- **rich removed** — CLI output now uses `click.echo`; rich is no longer a framework dependency
|
|
14
|
+
|
|
15
|
+
### Breaking changes
|
|
16
|
+
|
|
17
|
+
None. All new features are additive.
|
|
18
|
+
|
|
19
|
+
### Verification checklist
|
|
20
|
+
|
|
21
|
+
1. `uv run poe lint` — Ruff checks clean
|
|
22
|
+
1. `uv run poe typecheck` — mypy strict (125 source files, 0 errors)
|
|
23
|
+
1. `uv run poe test` — 790 passed, 20 skipped
|
|
24
|
+
1. `curl http://localhost:17000/health` — returns `{"status":"healthy"}`
|
|
25
|
+
|
|
26
|
+
______________________________________________________________________
|
|
27
|
+
|
|
3
28
|
## v0.5.0 — 2026-03-01
|
|
4
29
|
|
|
5
30
|
### Highlights
|
|
@@ -138,19 +138,6 @@ help = "Sync all dependency groups from uv.lock"
|
|
|
138
138
|
env = { UV_PROJECT_ENVIRONMENT = ".venv" }
|
|
139
139
|
|
|
140
140
|
# ── Local-only: requires sibling ../.github/ repo (never run in CI) ──────────
|
|
141
|
-
[tool.poe.tasks.install-hooks]
|
|
142
|
-
shell = """
|
|
143
|
-
if [[ ! -d ../.github ]]; then
|
|
144
|
-
echo "ERROR: ../.github repo not found. Run from within the DataEngineX workspace." >&2
|
|
145
|
-
exit 1
|
|
146
|
-
fi
|
|
147
|
-
cp ../.github/scripts/pre-commit-version-bump.sh .git/hooks/pre-commit
|
|
148
|
-
cp ../.github/scripts/bump-version.sh .git/hooks/bump-version.sh
|
|
149
|
-
chmod +x .git/hooks/pre-commit .git/hooks/bump-version.sh
|
|
150
|
-
echo "✓ pre-commit hook installed"
|
|
151
|
-
"""
|
|
152
|
-
help = "Install git hooks (auto-bump patch version on commit)"
|
|
153
|
-
|
|
154
141
|
[tool.poe.tasks.sync-claude-settings]
|
|
155
142
|
shell = """
|
|
156
143
|
if [[ ! -d ../.github ]]; then
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "dataenginex"
|
|
3
|
-
version = "1.1.
|
|
3
|
+
version = "1.1.1"
|
|
4
4
|
description = "DataEngineX - Core framework for AI/ML/Data engineering projects"
|
|
5
5
|
authors = [
|
|
6
6
|
{name = "Jay", email = "jayapal.myaka99@gmail.com"}
|
|
@@ -13,7 +13,6 @@ dependencies = [
|
|
|
13
13
|
"python-dotenv>=1.2.2",
|
|
14
14
|
"pyyaml>=6.0.3",
|
|
15
15
|
"click>=8.3.3",
|
|
16
|
-
"rich>=15.0.0",
|
|
17
16
|
"duckdb>=1.5.2",
|
|
18
17
|
"croniter>=6.2.2",
|
|
19
18
|
"httpx>=0.28.1",
|
|
@@ -36,7 +36,7 @@ from importlib.metadata import PackageNotFoundError, version
|
|
|
36
36
|
try:
|
|
37
37
|
__version__ = version("dataenginex")
|
|
38
38
|
except PackageNotFoundError:
|
|
39
|
-
__version__ = "1.1.
|
|
39
|
+
__version__ = "1.1.1"
|
|
40
40
|
|
|
41
41
|
# Re-export core symbols that don't require optional dependencies
|
|
42
42
|
from dataenginex.core import DataLayer, MedallionArchitecture, QualityGate
|
|
@@ -13,14 +13,20 @@ from pathlib import Path
|
|
|
13
13
|
|
|
14
14
|
import click
|
|
15
15
|
import structlog
|
|
16
|
-
from rich.console import Console
|
|
17
|
-
from rich.table import Table
|
|
18
16
|
|
|
19
17
|
from dataenginex.config.loader import load_config, validate_config
|
|
20
18
|
from dataenginex.core.exceptions import ConfigError
|
|
21
19
|
|
|
22
20
|
logger = structlog.get_logger()
|
|
23
|
-
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _print_table(title: str, rows: list[tuple[str, str]]) -> None:
|
|
24
|
+
col_w = max(len(r[0]) for r in rows) + 2
|
|
25
|
+
click.echo(f"\n{title}")
|
|
26
|
+
click.echo("-" * (col_w + 20))
|
|
27
|
+
for key, val in rows:
|
|
28
|
+
click.echo(f" {key:<{col_w}}{val}")
|
|
29
|
+
click.echo()
|
|
24
30
|
|
|
25
31
|
|
|
26
32
|
@click.group()
|
|
@@ -42,7 +48,7 @@ def validate(config_path: Path, overlay: Path | None) -> None:
|
|
|
42
48
|
try:
|
|
43
49
|
cfg = load_config(config_path, overlay=overlay)
|
|
44
50
|
except ConfigError as exc:
|
|
45
|
-
|
|
51
|
+
click.echo(f"Error: {exc}", err=True)
|
|
46
52
|
raise SystemExit(1) from exc
|
|
47
53
|
|
|
48
54
|
issues = validate_config(cfg)
|
|
@@ -50,30 +56,28 @@ def validate(config_path: Path, overlay: Path | None) -> None:
|
|
|
50
56
|
hard_errors = [i for i in issues if not i.startswith("Warning:")]
|
|
51
57
|
|
|
52
58
|
if warnings:
|
|
53
|
-
|
|
59
|
+
click.echo(f" {len(warnings)} warning(s):")
|
|
54
60
|
for w in warnings:
|
|
55
|
-
|
|
61
|
+
click.echo(f" ! {w}")
|
|
56
62
|
|
|
57
63
|
if hard_errors:
|
|
58
|
-
|
|
64
|
+
click.echo(f" {len(hard_errors)} error(s):", err=True)
|
|
59
65
|
for err in hard_errors:
|
|
60
|
-
|
|
66
|
+
click.echo(f" x {err}", err=True)
|
|
61
67
|
raise SystemExit(1)
|
|
62
68
|
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
console.print(table)
|
|
76
|
-
console.print("[green]Config is valid.[/green]")
|
|
69
|
+
_print_table(
|
|
70
|
+
f"Config: {cfg.project.name} v{cfg.project.version}",
|
|
71
|
+
[
|
|
72
|
+
("Data Sources", str(len(cfg.data.sources))),
|
|
73
|
+
("Data Pipelines", str(len(cfg.data.pipelines))),
|
|
74
|
+
("ML Experiments", str(len(cfg.ml.experiments))),
|
|
75
|
+
("AI Agents", str(len(cfg.ai.agents))),
|
|
76
|
+
("AI Collections", str(len(cfg.ai.collections))),
|
|
77
|
+
("Server", f"{cfg.server.host}:{cfg.server.port}"),
|
|
78
|
+
],
|
|
79
|
+
)
|
|
80
|
+
click.echo("Config is valid.")
|
|
77
81
|
|
|
78
82
|
|
|
79
83
|
@dex.command()
|
|
@@ -84,9 +88,9 @@ def version() -> None:
|
|
|
84
88
|
import sys
|
|
85
89
|
|
|
86
90
|
ver = importlib.metadata.version("dataenginex")
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
91
|
+
click.echo(f"DataEngineX {ver}")
|
|
92
|
+
click.echo(f"Python {sys.version}")
|
|
93
|
+
click.echo(f"Platform {platform.platform()}")
|
|
90
94
|
|
|
91
95
|
|
|
92
96
|
from dataenginex.cli.run import run # noqa: E402
|
|
@@ -6,14 +6,14 @@ from pathlib import Path
|
|
|
6
6
|
|
|
7
7
|
import click
|
|
8
8
|
import structlog
|
|
9
|
-
from rich.console import Console
|
|
10
|
-
from rich.table import Table
|
|
11
9
|
|
|
12
10
|
from dataenginex.config import load_config
|
|
13
11
|
from dataenginex.data.pipeline.runner import PipelineRunner
|
|
14
12
|
|
|
15
13
|
logger = structlog.get_logger()
|
|
16
|
-
|
|
14
|
+
|
|
15
|
+
_COLS = (24, 10, 10, 10, 8)
|
|
16
|
+
_HDR = ("Pipeline", "Status", "Rows In", "Rows Out", "Steps")
|
|
17
17
|
|
|
18
18
|
|
|
19
19
|
@click.command()
|
|
@@ -43,30 +43,27 @@ def run(
|
|
|
43
43
|
else:
|
|
44
44
|
raise click.UsageError("Specify a pipeline name or use --all")
|
|
45
45
|
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
table.add_column("Status")
|
|
50
|
-
table.add_column("Rows In")
|
|
51
|
-
table.add_column("Rows Out")
|
|
52
|
-
table.add_column("Steps")
|
|
46
|
+
click.echo("\nPipeline Results")
|
|
47
|
+
click.echo(" ".join(h.ljust(w) for h, w in zip(_HDR, _COLS, strict=True)))
|
|
48
|
+
click.echo("-" * (sum(_COLS) + 2 * len(_COLS)))
|
|
53
49
|
|
|
54
50
|
all_ok = True
|
|
55
51
|
for name, result in results.items():
|
|
56
|
-
status = "[green]OK[/green]" if result.success else "[red]FAIL[/red]"
|
|
57
52
|
if result.dry_run:
|
|
58
|
-
status = "
|
|
59
|
-
|
|
53
|
+
status = "DRY RUN"
|
|
54
|
+
elif result.success:
|
|
55
|
+
status = "OK"
|
|
56
|
+
else:
|
|
57
|
+
status = "FAIL"
|
|
60
58
|
all_ok = False
|
|
61
|
-
|
|
59
|
+
row_vals = [
|
|
62
60
|
name,
|
|
63
61
|
status,
|
|
64
62
|
str(result.rows_input),
|
|
65
63
|
str(result.rows_output),
|
|
66
64
|
str(result.steps_completed),
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
console.print(table)
|
|
65
|
+
]
|
|
66
|
+
click.echo(" ".join(v.ljust(w) for v, w in zip(row_vals, _COLS, strict=True)))
|
|
70
67
|
|
|
71
68
|
if not all_ok:
|
|
72
69
|
raise SystemExit(1)
|
|
@@ -41,27 +41,27 @@ def train(
|
|
|
41
41
|
model_dir: str,
|
|
42
42
|
) -> None:
|
|
43
43
|
"""Train ML models defined in dex.yaml."""
|
|
44
|
-
from rich.console import Console
|
|
45
|
-
|
|
46
44
|
from dataenginex.config.loader import load_config
|
|
47
45
|
|
|
48
|
-
console = Console()
|
|
49
46
|
config = load_config(Path(config_path))
|
|
50
47
|
|
|
51
48
|
if not config.ml.experiments:
|
|
52
|
-
|
|
49
|
+
click.echo("No experiments defined in config.")
|
|
53
50
|
return
|
|
54
51
|
|
|
55
52
|
if not experiment and not run_all:
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
console.print(f"Available: {available}")
|
|
53
|
+
click.echo("Specify an experiment name or use --all.")
|
|
54
|
+
click.echo(f"Available: {list(config.ml.experiments.keys())}")
|
|
59
55
|
return
|
|
60
56
|
|
|
61
57
|
experiments_to_run = _resolve_experiments(experiment, run_all, config)
|
|
62
58
|
|
|
63
|
-
|
|
64
|
-
|
|
59
|
+
rows = _run_experiments(experiments_to_run, config, model_dir)
|
|
60
|
+
click.echo("\nTraining Results")
|
|
61
|
+
click.echo(f" {'Experiment':<30}{'Status':<20}{'Score':<12}Version")
|
|
62
|
+
click.echo("-" * 72)
|
|
63
|
+
for row in rows:
|
|
64
|
+
click.echo(f" {row['name']:<30}{row['status']:<20}{row['score']:<12}{row['version']}")
|
|
65
65
|
|
|
66
66
|
|
|
67
67
|
def _resolve_experiments(
|
|
@@ -85,10 +85,8 @@ def _run_experiments(
|
|
|
85
85
|
experiments: dict[str, Any],
|
|
86
86
|
config: Any,
|
|
87
87
|
model_dir: str,
|
|
88
|
-
) ->
|
|
89
|
-
"""Run all experiments and return
|
|
90
|
-
from rich.table import Table
|
|
91
|
-
|
|
88
|
+
) -> list[dict[str, str]]:
|
|
89
|
+
"""Run all experiments and return rows for tabular display."""
|
|
92
90
|
from dataenginex.ml.registry import ModelRegistry
|
|
93
91
|
from dataenginex.ml.tracking import tracker_registry
|
|
94
92
|
|
|
@@ -102,12 +100,7 @@ def _run_experiments(
|
|
|
102
100
|
tracker = tracker_cls(**tracker_kwargs)
|
|
103
101
|
|
|
104
102
|
model_registry = ModelRegistry(persist_path=f"{model_dir}/registry.json")
|
|
105
|
-
|
|
106
|
-
results_table = Table(title="Training Results")
|
|
107
|
-
results_table.add_column("Experiment")
|
|
108
|
-
results_table.add_column("Status")
|
|
109
|
-
results_table.add_column("Score")
|
|
110
|
-
results_table.add_column("Version")
|
|
103
|
+
rows: list[dict[str, str]] = []
|
|
111
104
|
|
|
112
105
|
for exp_name, exp_config in experiments.items():
|
|
113
106
|
log = logger.bind(experiment=exp_name)
|
|
@@ -122,22 +115,19 @@ def _run_experiments(
|
|
|
122
115
|
model_dir,
|
|
123
116
|
log,
|
|
124
117
|
)
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
118
|
+
rows.append(
|
|
119
|
+
{
|
|
120
|
+
"name": exp_name,
|
|
121
|
+
"status": "OK",
|
|
122
|
+
"score": str(result.get("score", "N/A")),
|
|
123
|
+
"version": result.get("version", "?"),
|
|
124
|
+
}
|
|
130
125
|
)
|
|
131
126
|
except Exception as e:
|
|
132
127
|
log.error("training failed", error=str(e))
|
|
133
|
-
|
|
134
|
-
exp_name,
|
|
135
|
-
f"[red]FAIL: {e}[/red]",
|
|
136
|
-
"-",
|
|
137
|
-
"-",
|
|
138
|
-
)
|
|
128
|
+
rows.append({"name": exp_name, "status": f"FAIL: {e}", "score": "-", "version": "-"})
|
|
139
129
|
|
|
140
|
-
return
|
|
130
|
+
return rows
|
|
141
131
|
|
|
142
132
|
|
|
143
133
|
def _train_experiment(
|
|
@@ -807,7 +807,7 @@ wheels = [
|
|
|
807
807
|
|
|
808
808
|
[[package]]
|
|
809
809
|
name = "dataenginex"
|
|
810
|
-
version = "1.1.
|
|
810
|
+
version = "1.1.1"
|
|
811
811
|
source = { editable = "." }
|
|
812
812
|
dependencies = [
|
|
813
813
|
{ name = "arq" },
|
|
@@ -829,7 +829,6 @@ dependencies = [
|
|
|
829
829
|
{ name = "pyyaml" },
|
|
830
830
|
{ name = "qdrant-client" },
|
|
831
831
|
{ name = "redis" },
|
|
832
|
-
{ name = "rich" },
|
|
833
832
|
{ name = "structlog" },
|
|
834
833
|
{ name = "uvicorn" },
|
|
835
834
|
]
|
|
@@ -913,7 +912,6 @@ requires-dist = [
|
|
|
913
912
|
{ name = "pyyaml", specifier = ">=6.0.3" },
|
|
914
913
|
{ name = "qdrant-client", specifier = ">=1.17.1" },
|
|
915
914
|
{ name = "redis", specifier = ">=5.3.1" },
|
|
916
|
-
{ name = "rich", specifier = ">=15.0.0" },
|
|
917
915
|
{ name = "structlog", specifier = ">=25.5.0" },
|
|
918
916
|
{ name = "uvicorn", specifier = ">=0.46.0" },
|
|
919
917
|
]
|
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
name: Auto PR — feature → dev
|
|
2
|
-
|
|
3
|
-
on:
|
|
4
|
-
push:
|
|
5
|
-
branches:
|
|
6
|
-
- 'feature/**'
|
|
7
|
-
- 'fix/**'
|
|
8
|
-
|
|
9
|
-
permissions:
|
|
10
|
-
contents: write
|
|
11
|
-
pull-requests: write
|
|
12
|
-
|
|
13
|
-
jobs:
|
|
14
|
-
auto-pr:
|
|
15
|
-
uses: TheDataEngineX/.github/.github/workflows/auto-pr-to-dev.yml@main
|
|
16
|
-
secrets: inherit
|
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
name: Claude Code
|
|
2
|
-
|
|
3
|
-
on:
|
|
4
|
-
issue_comment:
|
|
5
|
-
types: [created]
|
|
6
|
-
pull_request_review_comment:
|
|
7
|
-
types: [created]
|
|
8
|
-
issues:
|
|
9
|
-
types: [opened, assigned]
|
|
10
|
-
pull_request_review:
|
|
11
|
-
types: [submitted]
|
|
12
|
-
|
|
13
|
-
permissions:
|
|
14
|
-
contents: read
|
|
15
|
-
pull-requests: read
|
|
16
|
-
issues: read
|
|
17
|
-
actions: read
|
|
18
|
-
id-token: write
|
|
19
|
-
|
|
20
|
-
jobs:
|
|
21
|
-
claude:
|
|
22
|
-
uses: TheDataEngineX/.github/.github/workflows/claude.yml@main
|
|
23
|
-
secrets: inherit
|