dataenginex 0.4.3__tar.gz → 0.4.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (290) hide show
  1. {dataenginex-0.4.3 → dataenginex-0.4.4}/.claude/commands/new-feature.md +1 -1
  2. {dataenginex-0.4.3 → dataenginex-0.4.4}/.claude/commands/validate.md +7 -6
  3. {dataenginex-0.4.3 → dataenginex-0.4.4}/.github/dependabot.yml +0 -2
  4. {dataenginex-0.4.3 → dataenginex-0.4.4}/.github/release-pr-template.md +1 -1
  5. dataenginex-0.4.4/.github/workflows/auto-pr.yml +16 -0
  6. {dataenginex-0.4.3 → dataenginex-0.4.4}/.github/workflows/ci.yml +6 -2
  7. {dataenginex-0.4.3 → dataenginex-0.4.4}/.github/workflows/release.yml +1 -1
  8. {dataenginex-0.4.3 → dataenginex-0.4.4}/.github/workflows/security.yml +3 -3
  9. {dataenginex-0.4.3 → dataenginex-0.4.4}/CLAUDE.md +6 -6
  10. {dataenginex-0.4.3 → dataenginex-0.4.4}/CONTRIBUTING.md +5 -5
  11. {dataenginex-0.4.3 → dataenginex-0.4.4}/PKG-INFO +1 -3
  12. {dataenginex-0.4.3 → dataenginex-0.4.4}/README.md +3 -3
  13. {dataenginex-0.4.3 → dataenginex-0.4.4}/docker-compose.test.yml +1 -1
  14. {dataenginex-0.4.3 → dataenginex-0.4.4}/docs/adr/0001-medallion-architecture.md +3 -3
  15. {dataenginex-0.4.3 → dataenginex-0.4.4}/docs/architecture.md +5 -8
  16. dataenginex-0.4.4/docs/ci-cd.md +97 -0
  17. {dataenginex-0.4.3 → dataenginex-0.4.4}/docs/contributing.md +4 -4
  18. {dataenginex-0.4.3 → dataenginex-0.4.4}/docs/development.md +6 -6
  19. {dataenginex-0.4.3 → dataenginex-0.4.4}/docs/quickstart.md +2 -2
  20. {dataenginex-0.4.3 → dataenginex-0.4.4}/docs/sdlc.md +12 -55
  21. {dataenginex-0.4.3 → dataenginex-0.4.4}/docs/security-scanning.md +2 -2
  22. {dataenginex-0.4.3 → dataenginex-0.4.4}/examples/01_hello_pipeline.py +1 -1
  23. {dataenginex-0.4.3 → dataenginex-0.4.4}/examples/02_api_quickstart.py +1 -1
  24. {dataenginex-0.4.3 → dataenginex-0.4.4}/examples/07_api_ingestion.py +1 -1
  25. {dataenginex-0.4.3 → dataenginex-0.4.4}/examples/08_spark_ml.py +3 -3
  26. {dataenginex-0.4.3 → dataenginex-0.4.4}/examples/09_feature_engineering.py +2 -2
  27. {dataenginex-0.4.3 → dataenginex-0.4.4}/examples/10_model_analysis.py +3 -3
  28. {dataenginex-0.4.3 → dataenginex-0.4.4}/examples/dashboard/dashboard_config.yaml +1 -1
  29. {dataenginex-0.4.3 → dataenginex-0.4.4}/examples/dashboard/run_dashboard.py +1 -1
  30. {dataenginex-0.4.3 → dataenginex-0.4.4}/poe_tasks.toml +2 -3
  31. {dataenginex-0.4.3 → dataenginex-0.4.4}/pyproject.toml +3 -4
  32. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/__init__.py +1 -1
  33. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/_json.py +8 -9
  34. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/ai/__init__.py +1 -35
  35. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/ai/memory/__init__.py +0 -3
  36. dataenginex-0.4.4/src/dataenginex/ai/observability/__init__.py +9 -0
  37. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/ai/observability/metrics.py +6 -6
  38. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/ai/runtime/__init__.py +0 -3
  39. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/ai/runtime/sandbox.py +1 -1
  40. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/api/pagination.py +1 -1
  41. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/core/exceptions.py +3 -46
  42. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/core/medallion_architecture.py +1 -1
  43. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/core/schemas.py +1 -1
  44. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/core/validators.py +105 -105
  45. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/data/connectors/dbt.py +1 -1
  46. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/data/connectors/spark.py +2 -2
  47. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/data/pipeline/__init__.py +0 -3
  48. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/data/registry.py +1 -1
  49. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/engine.py +1 -14
  50. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/lakehouse/partitioning.py +109 -109
  51. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/lakehouse/storage.py +1 -1
  52. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/ml/mlflow_registry.py +2 -2
  53. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/conftest.py +162 -162
  54. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/fixtures/sample_data.py +2 -2
  55. dataenginex-0.4.4/tests/integration/__init__.py +1 -0
  56. dataenginex-0.4.4/tests/unit/__init__.py +1 -0
  57. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_core_schemas_extended.py +1 -16
  58. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_llm.py +2 -2
  59. {dataenginex-0.4.3 → dataenginex-0.4.4}/uv.lock +77 -121
  60. dataenginex-0.4.3/.github/workflows/auto-pr.yml +0 -23
  61. dataenginex-0.4.3/.github/workflows/enforce-dev-to-main.yml +0 -11
  62. dataenginex-0.4.3/Dockerfile +0 -53
  63. dataenginex-0.4.3/docs/ci-cd.md +0 -361
  64. dataenginex-0.4.3/src/dataenginex/ai/observability/__init__.py +0 -15
  65. dataenginex-0.4.3/tests/integration/__init__.py +0 -1
  66. dataenginex-0.4.3/tests/unit/__init__.py +0 -1
  67. {dataenginex-0.4.3 → dataenginex-0.4.4}/.claude/settings.json +0 -0
  68. {dataenginex-0.4.3 → dataenginex-0.4.4}/.env.template +0 -0
  69. {dataenginex-0.4.3 → dataenginex-0.4.4}/.github/ISSUE_TEMPLATE/bug_report.yml +0 -0
  70. {dataenginex-0.4.3 → dataenginex-0.4.4}/.github/ISSUE_TEMPLATE/config.yml +0 -0
  71. {dataenginex-0.4.3 → dataenginex-0.4.4}/.github/ISSUE_TEMPLATE/feature_request.yml +0 -0
  72. {dataenginex-0.4.3 → dataenginex-0.4.4}/.github/PULL_REQUEST_TEMPLATE.md +0 -0
  73. {dataenginex-0.4.3 → dataenginex-0.4.4}/.github/labels.yml +0 -0
  74. {dataenginex-0.4.3 → dataenginex-0.4.4}/.github/workflows/docs-sync.yml +0 -0
  75. {dataenginex-0.4.3 → dataenginex-0.4.4}/.gitignore +0 -0
  76. {dataenginex-0.4.3 → dataenginex-0.4.4}/.gitleaks.toml +0 -0
  77. {dataenginex-0.4.3 → dataenginex-0.4.4}/.pre-commit-config.yaml +0 -0
  78. {dataenginex-0.4.3 → dataenginex-0.4.4}/.python-version +0 -0
  79. {dataenginex-0.4.3 → dataenginex-0.4.4}/CHANGELOG.md +0 -0
  80. {dataenginex-0.4.3 → dataenginex-0.4.4}/CODEOWNERS +0 -0
  81. {dataenginex-0.4.3 → dataenginex-0.4.4}/CODE_OF_CONDUCT.md +0 -0
  82. {dataenginex-0.4.3 → dataenginex-0.4.4}/LICENSE +0 -0
  83. {dataenginex-0.4.3 → dataenginex-0.4.4}/SECURITY.md +0 -0
  84. {dataenginex-0.4.3 → dataenginex-0.4.4}/docs/adr/0000-template.md +0 -0
  85. {dataenginex-0.4.3 → dataenginex-0.4.4}/docs/api-reference/ai.md +0 -0
  86. {dataenginex-0.4.3 → dataenginex-0.4.4}/docs/api-reference/api.md +0 -0
  87. {dataenginex-0.4.3 → dataenginex-0.4.4}/docs/api-reference/config.md +0 -0
  88. {dataenginex-0.4.3 → dataenginex-0.4.4}/docs/api-reference/core.md +0 -0
  89. {dataenginex-0.4.3 → dataenginex-0.4.4}/docs/api-reference/data.md +0 -0
  90. {dataenginex-0.4.3 → dataenginex-0.4.4}/docs/api-reference/engine.md +0 -0
  91. {dataenginex-0.4.3 → dataenginex-0.4.4}/docs/api-reference/index.md +0 -0
  92. {dataenginex-0.4.3 → dataenginex-0.4.4}/docs/api-reference/lakehouse.md +0 -0
  93. {dataenginex-0.4.3 → dataenginex-0.4.4}/docs/api-reference/middleware.md +0 -0
  94. {dataenginex-0.4.3 → dataenginex-0.4.4}/docs/api-reference/ml.md +0 -0
  95. {dataenginex-0.4.3 → dataenginex-0.4.4}/docs/api-reference/orchestration.md +0 -0
  96. {dataenginex-0.4.3 → dataenginex-0.4.4}/docs/api-reference/plugins.md +0 -0
  97. {dataenginex-0.4.3 → dataenginex-0.4.4}/docs/api-reference/secops.md +0 -0
  98. {dataenginex-0.4.3 → dataenginex-0.4.4}/docs/api-reference/warehouse.md +0 -0
  99. {dataenginex-0.4.3 → dataenginex-0.4.4}/docs/index.md +0 -0
  100. {dataenginex-0.4.3 → dataenginex-0.4.4}/docs/observability.md +0 -0
  101. {dataenginex-0.4.3 → dataenginex-0.4.4}/docs/release-notes.md +0 -0
  102. {dataenginex-0.4.3 → dataenginex-0.4.4}/examples/03_quality_gate.py +0 -0
  103. {dataenginex-0.4.3 → dataenginex-0.4.4}/examples/04_ml_training.py +0 -0
  104. {dataenginex-0.4.3 → dataenginex-0.4.4}/examples/05_rag_demo.py +0 -0
  105. {dataenginex-0.4.3 → dataenginex-0.4.4}/examples/06_llm_quickstart.py +0 -0
  106. {dataenginex-0.4.3 → dataenginex-0.4.4}/examples/GUIDE.md +0 -0
  107. {dataenginex-0.4.3 → dataenginex-0.4.4}/examples/data/events.csv +0 -0
  108. {dataenginex-0.4.3 → dataenginex-0.4.4}/examples/data/users.csv +0 -0
  109. {dataenginex-0.4.3 → dataenginex-0.4.4}/examples/dex.yaml +0 -0
  110. {dataenginex-0.4.3 → dataenginex-0.4.4}/examples/ecommerce/data/customers.csv +0 -0
  111. {dataenginex-0.4.3 → dataenginex-0.4.4}/examples/ecommerce/data/orders.csv +0 -0
  112. {dataenginex-0.4.3 → dataenginex-0.4.4}/examples/ecommerce/data/products.csv +0 -0
  113. {dataenginex-0.4.3 → dataenginex-0.4.4}/examples/ecommerce/dex.yaml +0 -0
  114. {dataenginex-0.4.3 → dataenginex-0.4.4}/examples/ecommerce/run_all.py +0 -0
  115. {dataenginex-0.4.3 → dataenginex-0.4.4}/examples/movies.csv +0 -0
  116. {dataenginex-0.4.3 → dataenginex-0.4.4}/scripts/localstack/init.sh +0 -0
  117. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/README.md +0 -0
  118. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/RELEASE_NOTES.md +0 -0
  119. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/ai/agents/__init__.py +0 -0
  120. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/ai/agents/builtin.py +0 -0
  121. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/ai/llm.py +0 -0
  122. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/ai/memory/base.py +0 -0
  123. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/ai/memory/episodic.py +0 -0
  124. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/ai/memory/long_term.py +0 -0
  125. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/ai/observability/audit.py +0 -0
  126. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/ai/observability/cost.py +0 -0
  127. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/ai/retrieval/__init__.py +0 -0
  128. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/ai/retrieval/builtin.py +0 -0
  129. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/ai/retrieval/graph.py +0 -0
  130. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/ai/routing/__init__.py +0 -0
  131. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/ai/routing/anthropic.py +0 -0
  132. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/ai/routing/guarded.py +0 -0
  133. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/ai/routing/ollama.py +0 -0
  134. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/ai/routing/openai.py +0 -0
  135. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/ai/routing/router.py +0 -0
  136. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/ai/runtime/checkpoint.py +0 -0
  137. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/ai/runtime/executor.py +0 -0
  138. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/ai/tools/__init__.py +0 -0
  139. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/ai/tools/builtin.py +0 -0
  140. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/ai/vectorstore.py +0 -0
  141. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/ai/workflows/__init__.py +0 -0
  142. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/ai/workflows/conditions.py +0 -0
  143. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/ai/workflows/dag.py +0 -0
  144. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/ai/workflows/human_loop.py +0 -0
  145. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/api/__init__.py +0 -0
  146. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/api/errors.py +0 -0
  147. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/api/schemas.py +0 -0
  148. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/cli/__init__.py +0 -0
  149. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/cli/main.py +0 -0
  150. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/cli/run.py +0 -0
  151. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/cli/secops.py +0 -0
  152. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/cli/train.py +0 -0
  153. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/config/__init__.py +0 -0
  154. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/config/defaults.py +0 -0
  155. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/config/loader.py +0 -0
  156. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/config/schema.py +0 -0
  157. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/config/settings.py +0 -0
  158. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/core/__init__.py +0 -0
  159. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/core/interfaces.py +0 -0
  160. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/core/quality.py +0 -0
  161. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/core/registry.py +0 -0
  162. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/data/__init__.py +0 -0
  163. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/data/connectors/__init__.py +0 -0
  164. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/data/connectors/_utils.py +0 -0
  165. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/data/connectors/csv.py +0 -0
  166. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/data/connectors/delta.py +0 -0
  167. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/data/connectors/duckdb.py +0 -0
  168. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/data/connectors/http.py +0 -0
  169. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/data/connectors/parquet.py +0 -0
  170. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/data/connectors/rest.py +0 -0
  171. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/data/connectors/sse.py +0 -0
  172. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/data/pipeline/dag.py +0 -0
  173. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/data/pipeline/run_history.py +0 -0
  174. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/data/pipeline/runner.py +0 -0
  175. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/data/profiler.py +0 -0
  176. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/data/quality/__init__.py +0 -0
  177. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/data/quality/gates.py +0 -0
  178. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/data/quality/spark.py +0 -0
  179. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/data/transforms/__init__.py +0 -0
  180. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/data/transforms/sql.py +0 -0
  181. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/lakehouse/__init__.py +0 -0
  182. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/lakehouse/catalog.py +0 -0
  183. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/middleware/__init__.py +0 -0
  184. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/middleware/domain_metrics.py +0 -0
  185. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/middleware/logging_config.py +0 -0
  186. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/middleware/metrics.py +0 -0
  187. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/ml/__init__.py +0 -0
  188. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/ml/drift.py +0 -0
  189. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/ml/features/__init__.py +0 -0
  190. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/ml/features/builtin.py +0 -0
  191. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/ml/metrics.py +0 -0
  192. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/ml/registry.py +0 -0
  193. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/ml/serving.py +0 -0
  194. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/ml/serving_engine/__init__.py +0 -0
  195. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/ml/serving_engine/builtin.py +0 -0
  196. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/ml/tracking/__init__.py +0 -0
  197. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/ml/tracking/builtin.py +0 -0
  198. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/ml/training.py +0 -0
  199. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/orchestration/__init__.py +0 -0
  200. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/orchestration/builtin.py +0 -0
  201. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/orchestration/scheduler.py +0 -0
  202. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/plugins/__init__.py +0 -0
  203. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/plugins/registry.py +0 -0
  204. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/py.typed +0 -0
  205. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/secops/__init__.py +0 -0
  206. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/secops/audit.py +0 -0
  207. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/secops/gate.py +0 -0
  208. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/secops/guard.py +0 -0
  209. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/secops/masking.py +0 -0
  210. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/secops/pii.py +0 -0
  211. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/store.py +0 -0
  212. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/warehouse/__init__.py +0 -0
  213. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/warehouse/lineage.py +0 -0
  214. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/warehouse/transforms.py +0 -0
  215. {dataenginex-0.4.3 → dataenginex-0.4.4}/src/dataenginex/worker.py +0 -0
  216. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/__init__.py +0 -0
  217. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/conformance/__init__.py +0 -0
  218. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/conformance/test_connector.py +0 -0
  219. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/conformance/test_feature_store.py +0 -0
  220. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/conformance/test_tracker.py +0 -0
  221. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/conformance/test_transform.py +0 -0
  222. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/fixtures/__init__.py +0 -0
  223. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/fixtures/sample_jobs.csv +0 -0
  224. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/fixtures/sample_jobs.json +0 -0
  225. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/integration/test_ai_integration.py +0 -0
  226. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/integration/test_cli_run.py +0 -0
  227. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/integration/test_config_cli.py +0 -0
  228. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/integration/test_lineage_integration.py +0 -0
  229. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/integration/test_ml_integration.py +0 -0
  230. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/integration/test_pipeline_e2e.py +0 -0
  231. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/integration/test_secops_integration.py +0 -0
  232. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/integration/test_storage_real.py +0 -0
  233. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/load/__init__.py +0 -0
  234. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_agent_runtime.py +0 -0
  235. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_ai_modules.py +0 -0
  236. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_api_pagination.py +0 -0
  237. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_api_schemas.py +0 -0
  238. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_api_validators.py +0 -0
  239. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_builtin_agent.py +0 -0
  240. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_builtin_feature_store.py +0 -0
  241. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_builtin_serving.py +0 -0
  242. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_builtin_tracker.py +0 -0
  243. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_cli_train.py +0 -0
  244. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_config_loader.py +0 -0
  245. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_config_schema.py +0 -0
  246. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_config_schema_extended.py +0 -0
  247. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_core_exceptions.py +0 -0
  248. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_core_interfaces.py +0 -0
  249. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_core_quality.py +0 -0
  250. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_core_registry.py +0 -0
  251. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_csv_connector.py +0 -0
  252. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_data.py +0 -0
  253. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_dbt_connector.py +0 -0
  254. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_dex_engine.py +0 -0
  255. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_dex_store.py +0 -0
  256. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_domain_metrics.py +0 -0
  257. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_domain_metrics_wiring.py +0 -0
  258. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_drift_scheduler.py +0 -0
  259. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_duckdb_connector.py +0 -0
  260. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_guarded_provider.py +0 -0
  261. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_lakehouse.py +0 -0
  262. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_llm_extended.py +0 -0
  263. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_llm_litellm_vllm.py +0 -0
  264. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_logging.py +0 -0
  265. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_medallion.py +0 -0
  266. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_medallion_extended.py +0 -0
  267. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_middleware_metrics.py +0 -0
  268. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_ml.py +0 -0
  269. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_parquet_connector.py +0 -0
  270. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_pipeline_dag.py +0 -0
  271. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_pipeline_runner.py +0 -0
  272. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_plugins.py +0 -0
  273. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_privacy_guard_wiring.py +0 -0
  274. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_quality_gates.py +0 -0
  275. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_quality_spark.py +0 -0
  276. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_retriever.py +0 -0
  277. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_retriever_graph.py +0 -0
  278. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_run_history.py +0 -0
  279. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_scheduler.py +0 -0
  280. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_secops.py +0 -0
  281. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_secops_engine_and_cli.py +0 -0
  282. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_secops_guard.py +0 -0
  283. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_spark_connector.py +0 -0
  284. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_spark_fixtures.py +0 -0
  285. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_sql_transforms.py +0 -0
  286. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_storage_abstraction.py +0 -0
  287. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_vectorstore.py +0 -0
  288. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_vectorstore_extended.py +0 -0
  289. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_warehouse.py +0 -0
  290. {dataenginex-0.4.3 → dataenginex-0.4.4}/tests/unit/test_warehouse_transforms.py +0 -0
@@ -1,4 +1,4 @@
1
- # New Feature (DEX)
1
+ # New Feature (DataEngineX)
2
2
 
3
3
  Scaffold a new feature for the `dataenginex` core framework. Ask for the feature name and description if not provided as $ARGUMENTS.
4
4
 
@@ -1,6 +1,6 @@
1
- # Validate (DEX)
1
+ # Validate (DataEngineX)
2
2
 
3
- Run the full DEX validation pipeline in order. Stop and report on first failure.
3
+ Run the full DataEngineX validation pipeline in order. Stop and report on first failure.
4
4
 
5
5
  1. **Lint**
6
6
 
@@ -25,10 +25,11 @@ Run the full DEX validation pipeline in order. Stop and report on first failure.
25
25
  ```bash
26
26
  uv run python examples/02_api_quickstart.py &
27
27
  sleep 2
28
- curl -sf http://localhost:17000/health | python -m json.tool
29
- curl -sf http://localhost:17000/ | python -m json.tool
30
- curl -sf http://localhost:17000/metrics | head -20
31
- curl -sf -X POST http://localhost:17000/echo -H "Content-Type: application/json" -d '{"message":"test"}' | python -m json.tool
28
+ # dataenginex has no built-in HTTP server; these endpoints apply to dex-studio
29
+ curl -sf http://localhost:7860/health | python -m json.tool
30
+ curl -sf http://localhost:7860/ | python -m json.tool
31
+ curl -sf http://localhost:7860/metrics | head -20
32
+ curl -sf -X POST http://localhost:7860/echo -H "Content-Type: application/json" -d '{"message":"test"}' | python -m json.tool
32
33
  ```
33
34
 
34
35
  Check response **bodies** — not just status codes. Kill the server after.
@@ -4,9 +4,7 @@ updates:
4
4
  directory: "/"
5
5
  schedule:
6
6
  interval: weekly
7
- target-branch: dev
8
7
  - package-ecosystem: github-actions
9
8
  directory: "/"
10
9
  schedule:
11
10
  interval: weekly
12
- target-branch: dev
@@ -1,6 +1,6 @@
1
1
  # Release PR Template
2
2
 
3
- Use this template whenever opening a release PR (`dev → main`). Reference `release-notes.md` and verify QA steps before merging.
3
+ Use this template whenever opening a release PR to `main`. Reference `release-notes.md` and verify QA steps before merging.
4
4
 
5
5
  ## Summary
6
6
 
@@ -0,0 +1,16 @@
1
+ name: Auto PR
2
+
3
+ on:
4
+ push:
5
+ branches:
6
+ - '*'
7
+
8
+ permissions:
9
+ contents: write
10
+ pull-requests: write
11
+
12
+ jobs:
13
+ auto-pr:
14
+ if: github.ref_name != 'main'
15
+ uses: TheDataEngineX/.github/.github/workflows/reusable-auto-pr-to-main.yml@main
16
+ secrets: inherit
@@ -2,9 +2,9 @@ name: CI
2
2
 
3
3
  on:
4
4
  push:
5
- branches: [main, dev]
5
+ branches: [main]
6
6
  pull_request:
7
- branches: [main, dev]
7
+ branches: [main]
8
8
  workflow_dispatch:
9
9
 
10
10
  schedule:
@@ -27,8 +27,12 @@ jobs:
27
27
  - run: uv sync --group ml
28
28
  env:
29
29
  UV_PROJECT_ENVIRONMENT: .venv
30
+ - uses: reviewdog/action-actionlint@v1
31
+
30
32
  - run: uv run poe quality
31
33
 
34
+ - run: uv build
35
+
32
36
  test:
33
37
  name: Tests (Python 3.13)
34
38
  runs-on: ubuntu-latest
@@ -80,6 +80,6 @@ jobs:
80
80
  RELEASE_TAG: ${{ inputs.tag || github.ref_name }}
81
81
  run: |
82
82
  gh release create "$RELEASE_TAG" \
83
- --title "DEX $RELEASE_TAG" \
83
+ --title "dataenginex $RELEASE_TAG" \
84
84
  --generate-notes \
85
85
  "sbom-dex-${{ steps.version.outputs.version }}.json"
@@ -2,9 +2,9 @@ name: Security Scans
2
2
 
3
3
  on:
4
4
  push:
5
- branches: [main, dev]
5
+ branches: [main]
6
6
  pull_request:
7
- branches: [main, dev]
7
+ branches: [main]
8
8
  workflow_dispatch:
9
9
 
10
10
  permissions:
@@ -13,5 +13,5 @@ permissions:
13
13
 
14
14
  jobs:
15
15
  security:
16
- uses: TheDataEngineX/.github/.github/workflows/security.yml@main
16
+ uses: TheDataEngineX/.github/.github/workflows/reusable-security.yml@main
17
17
  secrets: inherit
@@ -1,4 +1,4 @@
1
- # CLAUDE.md — DEX (dataenginex)
1
+ # CLAUDE.md — DataEngineX (dataenginex)
2
2
 
3
3
  Brief answers only. No explanations unless asked.
4
4
  Goal is to save Claude code tokens for lower cost without losing quality.
@@ -7,7 +7,7 @@ Goal is to save Claude code tokens for lower cost without losing quality.
7
7
 
8
8
  ## Project Overview
9
9
 
10
- **DEX** — unified Data + ML + AI library. Config-driven, self-hosted, local-first. Pure Python — no bundled HTTP server.
10
+ **DataEngineX** — unified Data + ML + AI library. Config-driven, self-hosted, local-first. Pure Python — no bundled HTTP server.
11
11
 
12
12
  | Package | Location | Purpose |
13
13
  |---------|----------|---------|
@@ -41,7 +41,7 @@ dex validate dex.yaml # Validate config file
41
41
  dex version # Show version + environment
42
42
 
43
43
  # Dev
44
- uv run poe dev # Dev server (uvicorn reload, port 17000) — for examples/API testing only
44
+ uv run poe dev # Dev server (uvicorn reload, port 17000) — for examples/API testing only (dataenginex has no built-in HTTP server; see dex-studio for web UI)
45
45
  uv run poe docker-up # Docker compose up
46
46
  uv run poe docker-down # Docker compose down
47
47
 
@@ -63,7 +63,7 @@ pip install 'litellm>=1.83.3' --no-deps # LLM routing (separate: pins python-dot
63
63
 
64
64
  ______________________________________________________________________
65
65
 
66
- ## Mandatory Validation
66
+ ## Validation
67
67
 
68
- Run `/validate` after ANY code change. Steps 4-5 (config validation + server smoke test) are NON-NEGOTIABLE.
69
- Tests passing ≠ app working.
68
+ After any code change run: `uv run poe check-all` (lint + typecheck + test).
69
+ Tests passing ≠ app working — run `dex validate dex.yaml` to verify config.
@@ -1,18 +1,18 @@
1
- # Contributing to DEX
1
+ # Contributing to DataEngineX
2
2
 
3
3
  Thank you for your interest in contributing to DataEngineX!
4
4
 
5
- **→ Full contributor guide: [docs/CONTRIBUTING.md](docs/CONTRIBUTING.md)**
5
+ **→ Full contributor guide: [docs/contributing.md](docs/contributing.md)**
6
6
 
7
7
  Quick essentials:
8
8
 
9
- - Development setup: [docs/DEVELOPMENT.md](docs/DEVELOPMENT.md)
10
- - Governance & legal: See [docs/CONTRIBUTING.md](docs/CONTRIBUTING.md)
9
+ - Development setup: [docs/development.md](docs/development.md)
10
+ - Governance & legal: See [docs/contributing.md](docs/contributing.md)
11
11
  - Code of Conduct: [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md)
12
12
 
13
13
  ## Quick Start
14
14
 
15
- 1. Fork and create a feature branch from `dev`
15
+ 1. Fork and create a feature branch from `main`
16
16
  1. Run local checks: `uv run poe check-all`
17
17
  1. Open a PR using `.github/PULL_REQUEST_TEMPLATE.md`
18
18
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dataenginex
3
- Version: 0.4.3
3
+ Version: 0.4.4
4
4
  Summary: DataEngineX — open-source, self-hosted, local-first Data + ML + AI workbench library
5
5
  Author-email: Jay <jayapal.myaka99@gmail.com>
6
6
  License: MIT License
@@ -30,7 +30,6 @@ Requires-Dist: click>=8.3.3
30
30
  Requires-Dist: croniter>=6.2.2
31
31
  Requires-Dist: duckdb>=1.5.2
32
32
  Requires-Dist: httpx>=0.28.1
33
- Requires-Dist: msgpack>=1.2.1
34
33
  Requires-Dist: orjson>=3.11.0
35
34
  Requires-Dist: prometheus-client>=0.25.0
36
35
  Requires-Dist: pyarrow>=23.0.1
@@ -38,7 +37,6 @@ Requires-Dist: pydantic-settings>=2.14.2
38
37
  Requires-Dist: pydantic>=2.13.4
39
38
  Requires-Dist: pyyaml>=6.0.3
40
39
  Requires-Dist: structlog>=25.5.0
41
- Requires-Dist: zstandard>=0.25.0
42
40
  Provides-Extra: cloud
43
41
  Requires-Dist: boto3>=1.43.7; extra == 'cloud'
44
42
  Requires-Dist: google-cloud-bigquery>=3.41.0; extra == 'cloud'
@@ -1,11 +1,11 @@
1
1
  # dataenginex
2
2
 
3
- [![CI](https://github.com/TheDataEngineX/dataenginex/actions/workflows/ci.yml/badge.svg?branch=dev)](https://github.com/TheDataEngineX/dataenginex/actions/workflows/ci.yml)
3
+ [![CI](https://github.com/TheDataEngineX/dataenginex/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/TheDataEngineX/dataenginex/actions/workflows/ci.yml)
4
4
  [![PyPI](https://img.shields.io/pypi/v/dataenginex)](https://pypi.org/project/dataenginex/)
5
5
  [![Python 3.13+](https://img.shields.io/badge/python-3.13+-blue.svg)](https://www.python.org/downloads/)
6
6
  [![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](LICENSE)
7
7
 
8
- The Python library that powers [DEX Studio](https://github.com/TheDataEngineX/dex-studio) — an open-source, self-hosted, local-first Data + ML + AI workbench for individuals and small teams. **Use the library directly when you want code; install DEX Studio when you want a UI.**
8
+ The Python library that powers [DataEngineX Studio](https://github.com/TheDataEngineX/dex-studio) — an open-source, self-hosted, local-first Data + ML + AI workbench for individuals and small teams. **Use the library directly when you want code; install DataEngineX Studio when you want a UI.**
9
9
 
10
10
  ______________________________________________________________________
11
11
 
@@ -116,7 +116,7 @@ git clone https://github.com/TheDataEngineX/dex-studio && cd dex-studio
116
116
  docker compose up # http://localhost:7860
117
117
  ```
118
118
 
119
- DEX Studio imports `dataenginex` directly — no separate API server, no HTTP hop.
119
+ DataEngineX Studio imports `dataenginex` directly — no separate API server, no HTTP hop.
120
120
 
121
121
  ______________________________________________________________________
122
122
 
@@ -1,5 +1,5 @@
1
1
  # ---------------------------------------------------------------
2
- # DEX — Storage Emulators for Integration Tests
2
+ # DataEngineX — Storage Emulators for Integration Tests
3
3
  #
4
4
  # Spins up LocalStack (S3) and fake-gcs-server (GCS) for local
5
5
  # integration testing without real cloud accounts.
@@ -8,18 +8,18 @@
8
8
 
9
9
  ## Context
10
10
 
11
- DEX needs a scalable, standardized data architecture that works across all projects (CareerDEX, Weather, etc.). The data pipeline needs to handle:
11
+ DataEngineX needs a scalable, standardized data architecture that works across all projects (CareerDEX, Weather, etc.). The data pipeline needs to handle:
12
12
 
13
13
  - Raw data ingestion from multiple sources
14
14
  - Data transformation and quality validation
15
15
  - Feature engineering for ML models
16
16
  - Analytics-ready data consumption
17
17
 
18
- Multiple projects will be built on the DEX platform, each with different data sources and transformations. We need a consistent, reusable pattern.
18
+ Multiple projects will be built on the DataEngineX platform, each with different data sources and transformations. We need a consistent, reusable pattern.
19
19
 
20
20
  ## Decision
21
21
 
22
- Implement the **Medallion Architecture** (Bronze → Silver → Gold layers) as the standard data architecture for all DEX projects.
22
+ Implement the **Medallion Architecture** (Bronze → Silver → Gold layers) as the standard data architecture for all DataEngineX projects.
23
23
 
24
24
  ### Architecture Layers
25
25
 
@@ -5,7 +5,7 @@
5
5
  **DataEngineX** is a unified Data + ML + AI **library** that wires industry tools through a
6
6
  single config-driven interface. One `dex.yaml` defines the entire project.
7
7
 
8
- **Design principle:** Pure Python library — no HTTP server bundled. Your application (DEX Studio,
8
+ **Design principle:** Pure Python library — no HTTP server bundled. Your application (DataEngineX Studio,
9
9
  your own FastAPI/Flask app, a script) imports `dataenginex` and owns the server layer.
10
10
 
11
11
  ## Architecture
@@ -56,9 +56,9 @@ dex.yaml
56
56
  Every subsystem follows the same pattern:
57
57
 
58
58
  1. **ABC** in `core/interfaces.py` — defines the contract (e.g. `BaseConnector`)
59
- 2. **BackendRegistry[T]** in `core/registry.py` — discovers and registers implementations
60
- 3. **Built-in** implements the ABC with zero external deps
61
- 4. **Extras** implement the same ABC, swapped in via config
59
+ 1. **BackendRegistry[T]** in `core/registry.py` — discovers and registers implementations
60
+ 1. **Built-in** implements the ABC with zero external deps
61
+ 1. **Extras** implement the same ABC, swapped in via config
62
62
 
63
63
  ```python
64
64
  from dataenginex.core.registry import BackendRegistry
@@ -108,10 +108,7 @@ DataEngineXError
108
108
  ├── ConfigError → ConfigValidationError
109
109
  ├── PipelineError → PipelineStepError
110
110
  ├── RegistryError
111
- ├── BackendNotInstalledError
112
- ├── TrainingError
113
- ├── ServingError
114
- └── AgentError → LLMProviderError
111
+ └── BackendNotInstalledError
115
112
  ```
116
113
 
117
114
  ## Module Map
@@ -0,0 +1,97 @@
1
+ # CI/CD Pipeline
2
+
3
+ **Continuous integration and release automation for dataenginex.**
4
+
5
+ ______________________________________________________________________
6
+
7
+ ## Overview
8
+
9
+ dataenginex is a pure Python library published to PyPI. The pipeline is:
10
+
11
+ - **CI**: Linting, type checking, testing, and security scanning on every PR to `main`.
12
+ - **Release**: Push a `v{X.Y.Z}` tag to `main` → `release.yml` builds, publishes to PyPI, and creates a GitHub Release.
13
+
14
+ ```mermaid
15
+ graph LR
16
+ Dev[Developer] --> PR[Open PR to main]
17
+ PR --> CI[CI: Lint/Test/Security]
18
+ CI --> Review[Code Review]
19
+ Review --> Merge["Merge to main"]
20
+ Merge --> Tag["Push tag vX.Y.Z"]
21
+ Tag --> Release[release.yml]
22
+ Release --> Build["Build wheel + sdist"]
23
+ Build --> PyPI["Publish to PyPI (OIDC)"]
24
+ Build --> GHRelease["GitHub Release + CycloneDX SBOM"]
25
+ ```
26
+
27
+ ______________________________________________________________________
28
+
29
+ ## Continuous Integration (CI)
30
+
31
+ **Workflow**: `.github/workflows/ci.yml`
32
+
33
+ **Triggers**:
34
+
35
+ - Push to `main`
36
+ - Pull requests targeting `main`
37
+
38
+ **Jobs**:
39
+
40
+ ### 1. Quality (`quality`)
41
+
42
+ ```bash
43
+ uv run ruff check src/ tests/
44
+ uv run mypy src/dataenginex/ --strict
45
+ ```
46
+
47
+ ### 2. Test (`test`)
48
+
49
+ ```bash
50
+ uv run pytest tests/ -x --tb=short --cov=src/dataenginex/
51
+ ```
52
+
53
+ Coverage threshold: 80%.
54
+
55
+ ### 3. Package validation
56
+
57
+ ```bash
58
+ uv build
59
+ ```
60
+
61
+ ### 4. Security Scans
62
+
63
+ Runs via the shared reusable workflow at `.github/workflows/security.yml`:
64
+
65
+ - **Trivy**: Misconfig and secret scan — results uploaded to GitHub Security tab.
66
+ - **CodeQL**: Handled by GitHub's default setup.
67
+
68
+ ______________________________________________________________________
69
+
70
+ ## Release Automation
71
+
72
+ **Workflow**: `.github/workflows/release.yml`
73
+
74
+ **Trigger**: Push a tag matching `v[0-9]+.[0-9]+.[0-9]+` to `main`.
75
+
76
+ **Jobs**:
77
+
78
+ 1. **build** — `uv build` → upload wheel + sdist
79
+ 1. **publish-pypi** — `pypa/gh-action-pypi-publish` (OIDC trusted publishing)
80
+ 1. **github-release** — CycloneDX SBOM → `gh release create`
81
+
82
+ **How to release**:
83
+
84
+ ```bash
85
+ git tag v1.2.3
86
+ git push origin v1.2.3
87
+ ```
88
+
89
+ ______________________________________________________________________
90
+
91
+ ## Workflows Overview
92
+
93
+ | Workflow | Trigger | Purpose |
94
+ | --- | --- | --- |
95
+ | **CI** | Push/PR to main | Lint + typecheck + test + security |
96
+ | **Security** | Push/PR to main | Trivy misconfig + secret scan |
97
+ | **Release** | Push tag `v*.*.*` to main | Build → PyPI → GitHub Release |
@@ -1,4 +1,4 @@
1
- # Contributing to DEX
1
+ # Contributing to DataEngineX
2
2
 
3
3
  Thank you for contributing!
4
4
 
@@ -6,7 +6,7 @@ Thank you for contributing!
6
6
 
7
7
  1. Read [development.md](./development.md) for setup instructions
8
8
  1. Fork the repository
9
- 1. Create a feature branch from `dev`
9
+ 1. Create a feature branch from `main`
10
10
  1. Make your changes
11
11
  1. Submit a pull request
12
12
 
@@ -68,7 +68,7 @@ Use semantic commit format:
68
68
 
69
69
  - This project is open source under MIT; keep license and attribution notices in redistributions.
70
70
  - Forks are welcome, but should use a distinct public name when redistributed as a separate project.
71
- - Do not present a fork as the official DataEngineX/DEX project.
71
+ - Do not present a fork as the official DataEngineX project.
72
72
  - See the project's license and brand guidelines for brand-usage details.
73
73
 
74
74
  ## Code Reviews
@@ -94,7 +94,7 @@ uv run poe check-all # Run all checks
94
94
  - `enhancement` - New feature or improvement
95
95
  - `good first issue` - Good for newcomers
96
96
  - `P1-high` / `P2-medium` - Priority levels
97
- - `dex-module` - Core DEX infrastructure
97
+ - `dex-module` - Core DataEngineX infrastructure
98
98
 
99
99
  ## Questions?
100
100
 
@@ -39,7 +39,7 @@ This installs all Python dependencies and configures pre-commit hooks.
39
39
  # 1. Clone repo and create feature branch
40
40
  git clone https://github.com/TheDataEngineX/dataenginex.git
41
41
  cd dataenginex
42
- git checkout -b feat/issue-XXX-description dev
42
+ git checkout -b my-feature-branch main
43
43
 
44
44
  # 2. Install Python deps & pre-commit hooks
45
45
  uv run poe setup
@@ -68,8 +68,8 @@ dataenginex/
68
68
  ### Branch & Commit
69
69
 
70
70
  ```bash
71
- # 1. Create feature branch from dev
72
- git checkout -b feat/issue-XXX-description dev
71
+ # 1. Create a feature branch from main
72
+ git checkout -b my-feature-branch main
73
73
 
74
74
  # 2. Make changes to src/
75
75
  # Add tests in tests/
@@ -83,7 +83,7 @@ uv run poe test
83
83
  git commit -m "feat(#XXX): description"
84
84
 
85
85
  # 5. Push & create PR
86
- git push origin feat/issue-XXX-description
86
+ git push origin my-feature-branch
87
87
  ```
88
88
 
89
89
  **PR Requirements:**
@@ -91,11 +91,11 @@ git push origin feat/issue-XXX-description
91
91
  - Link to issue: `Closes #XXX`
92
92
  - All checks pass (CI/CD ~3-5 min)
93
93
  - 1 approval required
94
- - Merge to `dev` when ready
94
+ - Merge to `main` when ready
95
95
 
96
96
  ### Version Management
97
97
 
98
- DEX has a single version source:
98
+ DataEngineX has a single version source:
99
99
 
100
100
  - **dataenginex version**: root `pyproject.toml` — managed automatically by the `release.yml` workflow
101
101
 
@@ -140,9 +140,9 @@ events = engine.secops_audit.events # list of AuditEvent
140
140
 
141
141
  ______________________________________________________________________
142
142
 
143
- ## DEX Studio
143
+ ## DataEngineX Studio
144
144
 
145
- DEX Studio is the optional web UI that loads a `dex.yaml` and provides a single
145
+ DataEngineX Studio is the optional web UI that loads a `dex.yaml` and provides a single
146
146
  control plane for Data / ML / AI / System. It uses `dataenginex` directly as a library
147
147
  — no separate server process needed.
148
148
 
@@ -6,7 +6,7 @@
6
6
 
7
7
  ______________________________________________________________________
8
8
 
9
- This document defines the software development lifecycle (SDLC) for DEX, including the required stages, artifacts, and quality gates.
9
+ This document defines the software development lifecycle (SDLC) for DataEngineX, including the required stages, artifacts, and quality gates.
10
10
 
11
11
  ## Goals
12
12
 
@@ -14,28 +14,6 @@ This document defines the software development lifecycle (SDLC) for DEX, includi
14
14
  - Enforce quality gates (lint, test, type checks, security scan) on every PR.
15
15
  - Maintain a complete audit trail for promotion across environments.
16
16
 
17
- ## Lifecycle Overview
18
-
19
- ```mermaid
20
- graph LR
21
- Plan[1. Plan] --> Design[2. Design]
22
- Design --> Implement[3. Implement]
23
- Implement --> Verify[4. Verify CI]
24
- Verify --> Release[5. Release CD]
25
- Release --> Operate[6. Operate]
26
- Operate -->|Issues| Plan
27
- Operate -->|Incidents| Rollback[Rollback]
28
- Rollback --> Operate
29
-
30
- style Plan fill:#e1f5ff
31
- style Design fill:#e1f5ff
32
- style Implement fill:#fff3cd
33
- style Verify fill:#fff3cd
34
- style Release fill:#d4edda
35
- style Operate fill:#d4edda
36
- style Rollback fill:#f8d7da
37
- ```
38
-
39
17
  ## Lifecycle Stages
40
18
 
41
19
  ### 1) Plan
@@ -89,7 +67,7 @@ ______________________________________________________________________
89
67
 
90
68
  ### 4) Verify (CI)
91
69
 
92
- **Entry**: Pull request opened against `dev` (integration) or `main` (release).
70
+ **Entry**: Pull request opened against `main`.
93
71
 
94
72
  **Required Checks**:
95
73
 
@@ -107,8 +85,7 @@ ______________________________________________________________________
107
85
 
108
86
  **Entry**:
109
87
 
110
- - Dev deployment: PR merged to `dev`.
111
- - Stage/Prod deployment: PR merged to `main`.
88
+ - PR merged to `main`.
112
89
 
113
90
  **Actions**:
114
91
 
@@ -141,49 +118,29 @@ ______________________________________________________________________
141
118
  ```mermaid
142
119
  flowchart TD
143
120
  Start([New Feature/Fix]) --> Issue[Create GitHub Issue]
144
- Issue --> Branch["Create branch: feat/xyz"]
121
+ Issue --> Branch["Create branch: my-feature"]
145
122
  Branch --> Code[Implement + Tests]
146
123
  Code --> Local["Local checks: lint, test, types"]
147
124
  Local -->|Failed| Code
148
- Local -->|Passed| PR["Open PR → dev"]
125
+ Local -->|Passed| PR["Open PR → main"]
149
126
  PR --> CI{CI Checks}
150
127
  CI -->|Failed| Fix[Fix Issues]
151
128
  Fix --> Code
152
129
  CI -->|Passed| Review{Code Review}
153
130
  Review -->|Changes Requested| Code
154
- Review -->|Approved| MergeDev["Merge to dev"]
155
- MergeDev --> DevDeploy[Auto-deploy to dev]
156
- DevDeploy --> DevTest{Dev Tests Pass?}
157
- DevTest -->|Failed| Hotfix[Hotfix or Rollback]
158
- Hotfix --> Code
159
- DevTest -->|Passed| ReleasePR["Create PR: dev → main"]
160
- ReleasePR --> ReleaseCI{CI Checks}
161
- ReleaseCI -->|Failed| Fix
162
- ReleaseCI -->|Passed| ReleaseReview{Code Review}
163
- ReleaseReview -->|Changes Requested| Code
164
- ReleaseReview -->|Approved| MergeMain["Merge to main"]
165
- MergeMain --> ProdDeploy[Auto-deploy to stage+prod]
166
- ProdDeploy --> Monitor[Monitor Production]
131
+ Review -->|Approved| MergeMain["Merge to main"]
132
+ MergeMain --> Prod[Auto-release if tagged]
133
+ Prod --> Monitor[Monitor]
167
134
  Monitor --> End([✓ Complete])
168
-
169
- style Start fill:#e1f5ff
170
- style Issue fill:#e1f5ff
171
- style Code fill:#fff3cd
172
- style CI fill:#fff3cd
173
- style Review fill:#fff3cd
174
- style DevDeploy fill:#d4edda
175
- style ProdDeploy fill:#d4edda
176
- style End fill:#d4edda
177
- style Hotfix fill:#f8d7da
178
135
  ```
179
136
 
180
137
  **Steps**:
181
138
 
182
- 1. Create or update a GitHub Issue and add it to the GitHub Project board.
183
- 1. Create a feature branch and implement changes.
184
- 1. Open a PR to `dev` and request review.
185
- 1. After validation in dev, open a release PR from `dev` → `main`.
139
+ 1. Create a feature branch from `main`.
140
+ 1. Implement changes, run local checks (`uv run poe check-all`).
141
+ 1. Open a PR to `main` and request review.
186
142
  1. Merge after all required checks pass.
143
+ 1. Push a `vX.Y.Z` tag to trigger a PyPI release.
187
144
 
188
145
  See [Contributing Guide](contributing.md) for branch naming, local checks, and PR conventions.
189
146
 
@@ -11,7 +11,7 @@ DataEngineX uses a multi-layered security scanning pipeline across CI/CD workflo
11
11
  | **CodeQL** | Static analysis (Python + GitHub Actions) | `security.yml` |
12
12
  | **pip-audit** | Python dependency vulnerability audit | `poe security` |
13
13
  | **CycloneDX** | SBOM generation (attached to releases) | `release-*.yml` |
14
- | **Dependabot** | Automated dependency updates | `.github/dependabot.yml` |
14
+ | **Renovate** | Automated dependency updates | `renovate.json` |
15
15
 
16
16
  ## How It Works
17
17
 
@@ -30,7 +30,7 @@ DataEngineX uses a multi-layered security scanning pipeline across CI/CD workflo
30
30
  trivy image dex:local
31
31
  ```
32
32
 
33
- ### On Release (release-dataenginex.yml)
33
+ ### On Release (release.yml)
34
34
 
35
35
  1. **CycloneDX SBOM** — generates a Software Bill of Materials in CycloneDX JSON format.
36
36
  1. **SBOM upload** — attaches the SBOM as a release asset on GitHub Releases.
@@ -1,5 +1,5 @@
1
1
  #!/usr/bin/env python
2
- """01_hello_pipeline.py — Minimal DEX pipeline walkthrough.
2
+ """01_hello_pipeline.py — Minimal DataEngineX pipeline walkthrough.
3
3
 
4
4
  Demonstrates:
5
5
  - Data profiling with ``DataProfiler``
@@ -44,7 +44,7 @@ def create_app() -> FastAPI:
44
44
  app = FastAPI(
45
45
  title="DataEngineX",
46
46
  version="0.8.9",
47
- description="Example DEX API instance",
47
+ description="Example DataEngineX API instance",
48
48
  )
49
49
 
50
50
  @app.get("/")
@@ -5,7 +5,7 @@ Demonstrates:
5
5
  - Fetching data from an external HTTP API (OpenWeatherMap-style)
6
6
  - Transforming raw responses into standardized records (Bronze → Silver)
7
7
  - Validating data quality before promotion (Silver → Gold)
8
- - Using DEX QualityGate for data validation
8
+ - Using DataEngineX QualityGate for data validation
9
9
 
10
10
  Run:
11
11
  uv run python examples/07_api_ingestion.py