dataenginex 1.0.2__tar.gz → 1.1.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (331) hide show
  1. dataenginex-1.1.1/.github/workflows/auto-pr.yml +23 -0
  2. {dataenginex-1.0.2 → dataenginex-1.1.1}/.github/workflows/docs-notify.yml +1 -1
  3. dataenginex-1.1.1/.github/workflows/release.yml +69 -0
  4. {dataenginex-1.0.2 → dataenginex-1.1.1}/.gitignore +3 -0
  5. {dataenginex-1.0.2 → dataenginex-1.1.1}/.gitleaks.toml +0 -4
  6. {dataenginex-1.0.2 → dataenginex-1.1.1}/.pre-commit-config.yaml +7 -12
  7. dataenginex-1.1.1/.release-please-manifest.json +3 -0
  8. dataenginex-1.1.1/CHANGELOG.md +540 -0
  9. {dataenginex-1.0.2 → dataenginex-1.1.1}/PKG-INFO +18 -15
  10. {dataenginex-1.0.2 → dataenginex-1.1.1}/README.md +1 -1
  11. {dataenginex-1.0.2 → dataenginex-1.1.1}/docs/release-notes.md +25 -0
  12. {dataenginex-1.0.2 → dataenginex-1.1.1}/examples/02_api_quickstart.py +1 -2
  13. {dataenginex-1.0.2 → dataenginex-1.1.1}/examples/07_api_ingestion.py +2 -1
  14. {dataenginex-1.0.2 → dataenginex-1.1.1}/examples/08_spark_ml.py +1 -0
  15. {dataenginex-1.0.2 → dataenginex-1.1.1}/examples/09_feature_engineering.py +1 -0
  16. {dataenginex-1.0.2 → dataenginex-1.1.1}/examples/10_model_analysis.py +1 -0
  17. dataenginex-1.1.1/examples/data/events.csv +11 -0
  18. dataenginex-1.1.1/examples/data/users.csv +11 -0
  19. {dataenginex-1.0.2 → dataenginex-1.1.1}/examples/dex.yaml +4 -4
  20. dataenginex-1.1.1/examples/ecommerce/data/customers.csv +21 -0
  21. dataenginex-1.1.1/examples/ecommerce/data/orders.csv +51 -0
  22. dataenginex-1.1.1/examples/ecommerce/data/products.csv +11 -0
  23. dataenginex-1.1.1/examples/ecommerce/dex.yaml +142 -0
  24. dataenginex-1.1.1/examples/ecommerce/run_all.py +426 -0
  25. {dataenginex-1.0.2 → dataenginex-1.1.1}/poe_tasks.toml +3 -20
  26. {dataenginex-1.0.2 → dataenginex-1.1.1}/pyproject.toml +40 -44
  27. {dataenginex-1.0.2 → dataenginex-1.1.1}/sonar-project.properties +1 -1
  28. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/__init__.py +1 -1
  29. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/ai/observability/audit.py +11 -0
  30. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/api/factory.py +18 -9
  31. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/api/health.py +17 -24
  32. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/api/routers/ai.py +64 -1
  33. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/api/routers/data.py +24 -0
  34. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/api/routers/ml.py +13 -1
  35. dataenginex-1.1.1/src/dataenginex/api/routers/pipelines.py +194 -0
  36. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/api/scim.py +240 -2
  37. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/cli/main.py +29 -25
  38. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/cli/run.py +14 -17
  39. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/cli/train.py +21 -31
  40. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/data/connectors/csv.py +11 -5
  41. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/data/connectors/legacy.py +2 -0
  42. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/data/transforms/sql.py +25 -0
  43. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/ml/__init__.py +3 -3
  44. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/ml/training.py +8 -0
  45. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/ml/vectorstore.py +105 -80
  46. dataenginex-1.1.1/src/dataenginex/warehouse/lineage.py +455 -0
  47. dataenginex-1.1.1/src/dataenginex/worker.py +180 -0
  48. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/integration/test_api_middleware_integration.py +1 -1
  49. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/integration/test_full_app.py +1 -1
  50. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_api_factory.py +7 -6
  51. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_data_router.py +22 -0
  52. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_pipeline_router_extended.py +19 -14
  53. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_sql_transforms.py +43 -10
  54. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_vectorstore.py +13 -8
  55. {dataenginex-1.0.2 → dataenginex-1.1.1}/uv.lock +668 -768
  56. dataenginex-1.0.2/.github/workflows/auto-pr-dev-to-main.yml +0 -15
  57. dataenginex-1.0.2/.github/workflows/auto-pr-to-dev.yml +0 -16
  58. dataenginex-1.0.2/.github/workflows/claude.yml +0 -23
  59. dataenginex-1.0.2/.github/workflows/pypi-publish.yml +0 -290
  60. dataenginex-1.0.2/.github/workflows/release-dex.yml +0 -150
  61. dataenginex-1.0.2/.github/workflows/release-please.yml +0 -16
  62. dataenginex-1.0.2/.release-please-manifest.json +0 -3
  63. dataenginex-1.0.2/CHANGELOG.md +0 -285
  64. dataenginex-1.0.2/docs/roadmap/project-roadmap.csv +0 -37
  65. dataenginex-1.0.2/docs/roadmap/project-roadmap.json +0 -11
  66. dataenginex-1.0.2/docs/superpowers/plans/2026-03-21-phase-0-foundation.md +0 -2242
  67. dataenginex-1.0.2/docs/superpowers/plans/2026-03-22-dataenginex-1.0-master-plan.md +0 -490
  68. dataenginex-1.0.2/docs/superpowers/plans/2026-03-22-phase-1-data-layer.md +0 -2016
  69. dataenginex-1.0.2/docs/superpowers/plans/2026-03-22-phase-6a-dex-engine-integration.md +0 -2235
  70. dataenginex-1.0.2/docs/superpowers/plans/2026-03-22-phase-6b-dex-studio-redesign.md +0 -2050
  71. dataenginex-1.0.2/docs/superpowers/plans/2026-03-23-careerdex-example.md +0 -1006
  72. dataenginex-1.0.2/docs/superpowers/plans/2026-03-23-dex-studio-direct-import.md +0 -1645
  73. dataenginex-1.0.2/docs/superpowers/plans/2026-03-24-dex-naming-architecture-dry.md +0 -1093
  74. dataenginex-1.0.2/docs/superpowers/plans/2026-03-28-unified-docs-and-tooling.md +0 -2137
  75. dataenginex-1.0.2/docs/superpowers/specs/2026-03-21-dataenginex-v2-system-redesign.md +0 -853
  76. dataenginex-1.0.2/docs/superpowers/specs/2026-03-22-phase-6-integration-design.md +0 -596
  77. dataenginex-1.0.2/docs/superpowers/specs/2026-03-23-dex-studio-direct-import-design.md +0 -469
  78. dataenginex-1.0.2/docs/superpowers/specs/2026-03-24-dex-naming-architecture-dry-design.md +0 -412
  79. dataenginex-1.0.2/docs/superpowers/specs/2026-03-28-unified-docs-and-tooling-design.md +0 -683
  80. dataenginex-1.0.2/scripts/GUIDE.md +0 -128
  81. dataenginex-1.0.2/scripts/localstack/create-buckets.sh +0 -4
  82. dataenginex-1.0.2/scripts/promote.sh +0 -125
  83. dataenginex-1.0.2/src/dataenginex/api/routers/pipelines.py +0 -80
  84. dataenginex-1.0.2/src/dataenginex/dashboard/__init__.py +0 -29
  85. dataenginex-1.0.2/src/dataenginex/dashboard/app.py +0 -165
  86. dataenginex-1.0.2/src/dataenginex/dashboard/panels.py +0 -129
  87. dataenginex-1.0.2/src/dataenginex/warehouse/lineage.py +0 -204
  88. dataenginex-1.0.2/tasks/findings.md +0 -71
  89. dataenginex-1.0.2/tasks/lessons.md +0 -63
  90. {dataenginex-1.0.2 → dataenginex-1.1.1}/.claude/commands/new-feature.md +0 -0
  91. {dataenginex-1.0.2 → dataenginex-1.1.1}/.claude/commands/validate.md +0 -0
  92. {dataenginex-1.0.2 → dataenginex-1.1.1}/.claude/settings.json +0 -0
  93. {dataenginex-1.0.2 → dataenginex-1.1.1}/.env.template +0 -0
  94. {dataenginex-1.0.2 → dataenginex-1.1.1}/.github/ISSUE_TEMPLATE/bug_report.yml +0 -0
  95. {dataenginex-1.0.2 → dataenginex-1.1.1}/.github/ISSUE_TEMPLATE/config.yml +0 -0
  96. {dataenginex-1.0.2 → dataenginex-1.1.1}/.github/ISSUE_TEMPLATE/feature_request.yml +0 -0
  97. {dataenginex-1.0.2 → dataenginex-1.1.1}/.github/PULL_REQUEST_TEMPLATE.md +0 -0
  98. {dataenginex-1.0.2 → dataenginex-1.1.1}/.github/dependabot.yml +0 -0
  99. {dataenginex-1.0.2 → dataenginex-1.1.1}/.github/labels.yml +0 -0
  100. {dataenginex-1.0.2 → dataenginex-1.1.1}/.github/release-pr-template.md +0 -0
  101. {dataenginex-1.0.2 → dataenginex-1.1.1}/.github/workflows/ci.yml +0 -0
  102. {dataenginex-1.0.2 → dataenginex-1.1.1}/.github/workflows/docker-build-push.yml +0 -0
  103. {dataenginex-1.0.2 → dataenginex-1.1.1}/.github/workflows/enforce-dev-to-main.yml +0 -0
  104. {dataenginex-1.0.2 → dataenginex-1.1.1}/.github/workflows/label-sync.yml +0 -0
  105. {dataenginex-1.0.2 → dataenginex-1.1.1}/.github/workflows/security.yml +0 -0
  106. {dataenginex-1.0.2 → dataenginex-1.1.1}/.python-version +0 -0
  107. {dataenginex-1.0.2 → dataenginex-1.1.1}/CLAUDE.md +0 -0
  108. {dataenginex-1.0.2 → dataenginex-1.1.1}/CODEOWNERS +0 -0
  109. {dataenginex-1.0.2 → dataenginex-1.1.1}/CODE_OF_CONDUCT.md +0 -0
  110. {dataenginex-1.0.2 → dataenginex-1.1.1}/CONTRIBUTING.md +0 -0
  111. {dataenginex-1.0.2 → dataenginex-1.1.1}/Dockerfile +0 -0
  112. {dataenginex-1.0.2 → dataenginex-1.1.1}/LICENSE +0 -0
  113. {dataenginex-1.0.2 → dataenginex-1.1.1}/SECURITY.md +0 -0
  114. {dataenginex-1.0.2 → dataenginex-1.1.1}/docker-compose.test.yml +0 -0
  115. {dataenginex-1.0.2 → dataenginex-1.1.1}/docs/adr/0000-template.md +0 -0
  116. {dataenginex-1.0.2 → dataenginex-1.1.1}/docs/adr/0001-medallion-architecture.md +0 -0
  117. {dataenginex-1.0.2 → dataenginex-1.1.1}/docs/api-reference/api.md +0 -0
  118. {dataenginex-1.0.2 → dataenginex-1.1.1}/docs/api-reference/core.md +0 -0
  119. {dataenginex-1.0.2 → dataenginex-1.1.1}/docs/api-reference/dashboard.md +0 -0
  120. {dataenginex-1.0.2 → dataenginex-1.1.1}/docs/api-reference/data.md +0 -0
  121. {dataenginex-1.0.2 → dataenginex-1.1.1}/docs/api-reference/index.md +0 -0
  122. {dataenginex-1.0.2 → dataenginex-1.1.1}/docs/api-reference/lakehouse.md +0 -0
  123. {dataenginex-1.0.2 → dataenginex-1.1.1}/docs/api-reference/middleware.md +0 -0
  124. {dataenginex-1.0.2 → dataenginex-1.1.1}/docs/api-reference/ml.md +0 -0
  125. {dataenginex-1.0.2 → dataenginex-1.1.1}/docs/api-reference/plugins.md +0 -0
  126. {dataenginex-1.0.2 → dataenginex-1.1.1}/docs/api-reference/warehouse.md +0 -0
  127. {dataenginex-1.0.2 → dataenginex-1.1.1}/docs/architecture.md +0 -0
  128. {dataenginex-1.0.2 → dataenginex-1.1.1}/docs/ci-cd.md +0 -0
  129. {dataenginex-1.0.2 → dataenginex-1.1.1}/docs/contributing.md +0 -0
  130. {dataenginex-1.0.2 → dataenginex-1.1.1}/docs/development.md +0 -0
  131. {dataenginex-1.0.2 → dataenginex-1.1.1}/docs/index.md +0 -0
  132. {dataenginex-1.0.2 → dataenginex-1.1.1}/docs/observability.md +0 -0
  133. {dataenginex-1.0.2 → dataenginex-1.1.1}/docs/quickstart.md +0 -0
  134. {dataenginex-1.0.2 → dataenginex-1.1.1}/docs/sdlc.md +0 -0
  135. {dataenginex-1.0.2 → dataenginex-1.1.1}/docs/security-scanning.md +0 -0
  136. {dataenginex-1.0.2 → dataenginex-1.1.1}/examples/01_hello_pipeline.py +0 -0
  137. {dataenginex-1.0.2 → dataenginex-1.1.1}/examples/03_quality_gate.py +0 -0
  138. {dataenginex-1.0.2 → dataenginex-1.1.1}/examples/04_ml_training.py +0 -0
  139. {dataenginex-1.0.2 → dataenginex-1.1.1}/examples/05_rag_demo.py +0 -0
  140. {dataenginex-1.0.2 → dataenginex-1.1.1}/examples/06_llm_quickstart.py +0 -0
  141. {dataenginex-1.0.2 → dataenginex-1.1.1}/examples/GUIDE.md +0 -0
  142. {dataenginex-1.0.2 → dataenginex-1.1.1}/examples/dashboard/dashboard_config.yaml +0 -0
  143. {dataenginex-1.0.2 → dataenginex-1.1.1}/examples/dashboard/run_dashboard.py +0 -0
  144. {dataenginex-1.0.2 → dataenginex-1.1.1}/examples/movies.csv +0 -0
  145. {dataenginex-1.0.2 → dataenginex-1.1.1}/release-please-config.json +0 -0
  146. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/README.md +0 -0
  147. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/RELEASE_NOTES.md +0 -0
  148. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/ai/__init__.py +0 -0
  149. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/ai/agents/__init__.py +0 -0
  150. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/ai/agents/builtin.py +0 -0
  151. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/ai/memory/__init__.py +0 -0
  152. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/ai/memory/base.py +0 -0
  153. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/ai/memory/episodic.py +0 -0
  154. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/ai/memory/long_term.py +0 -0
  155. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/ai/observability/__init__.py +0 -0
  156. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/ai/observability/cost.py +0 -0
  157. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/ai/observability/langfuse.py +0 -0
  158. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/ai/observability/metrics.py +0 -0
  159. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/ai/retrieval/__init__.py +0 -0
  160. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/ai/retrieval/builtin.py +0 -0
  161. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/ai/retrieval/graph.py +0 -0
  162. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/ai/routing/__init__.py +0 -0
  163. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/ai/routing/anthropic.py +0 -0
  164. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/ai/routing/huggingface.py +0 -0
  165. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/ai/routing/ollama.py +0 -0
  166. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/ai/routing/openai.py +0 -0
  167. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/ai/routing/router.py +0 -0
  168. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/ai/runtime/__init__.py +0 -0
  169. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/ai/runtime/checkpoint.py +0 -0
  170. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/ai/runtime/executor.py +0 -0
  171. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/ai/runtime/sandbox.py +0 -0
  172. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/ai/tools/__init__.py +0 -0
  173. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/ai/tools/builtin.py +0 -0
  174. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/ai/workflows/__init__.py +0 -0
  175. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/ai/workflows/conditions.py +0 -0
  176. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/ai/workflows/dag.py +0 -0
  177. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/ai/workflows/human_loop.py +0 -0
  178. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/api/__init__.py +0 -0
  179. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/api/auth.py +0 -0
  180. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/api/errors.py +0 -0
  181. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/api/jwks.py +0 -0
  182. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/api/ldap_sync.py +0 -0
  183. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/api/pagination.py +0 -0
  184. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/api/rate_limit.py +0 -0
  185. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/api/rbac.py +0 -0
  186. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/api/routers/__init__.py +0 -0
  187. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/api/routers/health.py +0 -0
  188. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/api/routers/root.py +0 -0
  189. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/api/routers/system.py +0 -0
  190. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/api/schemas.py +0 -0
  191. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/cli/__init__.py +0 -0
  192. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/cli/serve.py +0 -0
  193. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/config/__init__.py +0 -0
  194. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/config/defaults.py +0 -0
  195. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/config/loader.py +0 -0
  196. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/config/schema.py +0 -0
  197. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/core/__init__.py +0 -0
  198. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/core/exceptions.py +0 -0
  199. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/core/interfaces.py +0 -0
  200. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/core/medallion_architecture.py +0 -0
  201. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/core/quality.py +0 -0
  202. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/core/registry.py +0 -0
  203. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/core/schemas.py +0 -0
  204. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/core/validators.py +0 -0
  205. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/data/__init__.py +0 -0
  206. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/data/connectors/__init__.py +0 -0
  207. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/data/connectors/duckdb.py +0 -0
  208. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/data/pipeline/__init__.py +0 -0
  209. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/data/pipeline/dag.py +0 -0
  210. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/data/pipeline/run_history.py +0 -0
  211. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/data/pipeline/runner.py +0 -0
  212. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/data/profiler.py +0 -0
  213. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/data/quality/__init__.py +0 -0
  214. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/data/quality/gates.py +0 -0
  215. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/data/quality/spark.py +0 -0
  216. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/data/registry.py +0 -0
  217. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/data/transforms/__init__.py +0 -0
  218. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/lakehouse/__init__.py +0 -0
  219. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/lakehouse/catalog.py +0 -0
  220. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/lakehouse/partitioning.py +0 -0
  221. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/lakehouse/storage.py +0 -0
  222. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/middleware/__init__.py +0 -0
  223. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/middleware/domain_metrics.py +0 -0
  224. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/middleware/logging_config.py +0 -0
  225. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/middleware/metrics.py +0 -0
  226. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/middleware/metrics_middleware.py +0 -0
  227. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/middleware/request_logging.py +0 -0
  228. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/middleware/tracing.py +0 -0
  229. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/ml/drift.py +0 -0
  230. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/ml/features/__init__.py +0 -0
  231. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/ml/features/builtin.py +0 -0
  232. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/ml/llm.py +0 -0
  233. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/ml/metrics.py +0 -0
  234. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/ml/mlflow_registry.py +0 -0
  235. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/ml/registry.py +0 -0
  236. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/ml/scheduler.py +0 -0
  237. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/ml/serving.py +0 -0
  238. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/ml/serving_engine/__init__.py +0 -0
  239. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/ml/serving_engine/builtin.py +0 -0
  240. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/ml/tracking/__init__.py +0 -0
  241. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/ml/tracking/builtin.py +0 -0
  242. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/orchestration/__init__.py +0 -0
  243. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/orchestration/builtin.py +0 -0
  244. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/plugins/__init__.py +0 -0
  245. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/plugins/registry.py +0 -0
  246. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/py.typed +0 -0
  247. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/secops/__init__.py +0 -0
  248. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/secops/audit.py +0 -0
  249. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/secops/gate.py +0 -0
  250. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/secops/masking.py +0 -0
  251. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/secops/pii.py +0 -0
  252. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/warehouse/__init__.py +0 -0
  253. {dataenginex-1.0.2 → dataenginex-1.1.1}/src/dataenginex/warehouse/transforms.py +0 -0
  254. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/__init__.py +0 -0
  255. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/conformance/__init__.py +0 -0
  256. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/conformance/test_connector.py +0 -0
  257. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/conformance/test_feature_store.py +0 -0
  258. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/conformance/test_tracker.py +0 -0
  259. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/conformance/test_transform.py +0 -0
  260. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/conftest.py +0 -0
  261. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/fixtures/__init__.py +0 -0
  262. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/fixtures/sample_data.py +0 -0
  263. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/fixtures/sample_jobs.csv +0 -0
  264. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/fixtures/sample_jobs.json +0 -0
  265. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/integration/__init__.py +0 -0
  266. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/integration/test_ai_integration.py +0 -0
  267. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/integration/test_cli_run.py +0 -0
  268. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/integration/test_config_cli.py +0 -0
  269. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/integration/test_lineage_integration.py +0 -0
  270. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/integration/test_ml_integration.py +0 -0
  271. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/integration/test_pipeline_e2e.py +0 -0
  272. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/integration/test_secops_integration.py +0 -0
  273. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/integration/test_storage_real.py +0 -0
  274. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/load/__init__.py +0 -0
  275. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/__init__.py +0 -0
  276. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_agent_runtime.py +0 -0
  277. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_ai_modules.py +0 -0
  278. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_ai_router.py +0 -0
  279. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_ai_router_extended.py +0 -0
  280. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_api_auth.py +0 -0
  281. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_api_jwks.py +0 -0
  282. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_api_pagination.py +0 -0
  283. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_api_rate_limit.py +0 -0
  284. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_api_rbac.py +0 -0
  285. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_api_schemas.py +0 -0
  286. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_api_scim.py +0 -0
  287. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_api_validators.py +0 -0
  288. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_builtin_agent.py +0 -0
  289. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_builtin_feature_store.py +0 -0
  290. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_builtin_serving.py +0 -0
  291. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_builtin_tracker.py +0 -0
  292. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_cli_train.py +0 -0
  293. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_config_loader.py +0 -0
  294. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_config_schema.py +0 -0
  295. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_core_exceptions.py +0 -0
  296. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_core_interfaces.py +0 -0
  297. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_core_registry.py +0 -0
  298. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_csv_connector.py +0 -0
  299. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_data.py +0 -0
  300. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_domain_metrics.py +0 -0
  301. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_domain_metrics_wiring.py +0 -0
  302. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_drift_scheduler.py +0 -0
  303. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_duckdb_connector.py +0 -0
  304. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_errors.py +0 -0
  305. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_health.py +0 -0
  306. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_lakehouse.py +0 -0
  307. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_llm.py +0 -0
  308. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_llm_litellm_vllm.py +0 -0
  309. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_logging.py +0 -0
  310. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_medallion.py +0 -0
  311. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_metrics.py +0 -0
  312. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_middleware.py +0 -0
  313. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_ml.py +0 -0
  314. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_ml_router.py +0 -0
  315. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_observability_langfuse.py +0 -0
  316. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_pipeline_dag.py +0 -0
  317. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_pipeline_runner.py +0 -0
  318. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_plugins.py +0 -0
  319. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_quality_gates.py +0 -0
  320. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_quality_spark.py +0 -0
  321. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_retriever.py +0 -0
  322. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_retriever_graph.py +0 -0
  323. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_run_history.py +0 -0
  324. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_scheduler.py +0 -0
  325. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_secops.py +0 -0
  326. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_security_extended.py +0 -0
  327. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_spark_fixtures.py +0 -0
  328. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_storage_abstraction.py +0 -0
  329. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_system_router.py +0 -0
  330. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_tracing.py +0 -0
  331. {dataenginex-1.0.2 → dataenginex-1.1.1}/tests/unit/test_warehouse.py +0 -0
@@ -0,0 +1,23 @@
1
+ name: Auto PR
2
+
3
+ on:
4
+ push:
5
+ branches:
6
+ - dev
7
+ - 'feature/**'
8
+ - 'fix/**'
9
+
10
+ permissions:
11
+ contents: write
12
+ pull-requests: write
13
+
14
+ jobs:
15
+ feature-to-dev:
16
+ if: startsWith(github.ref, 'refs/heads/feature/') || startsWith(github.ref, 'refs/heads/fix/')
17
+ uses: TheDataEngineX/.github/.github/workflows/auto-pr-to-dev.yml@main
18
+ secrets: inherit
19
+
20
+ dev-to-main:
21
+ if: github.ref == 'refs/heads/dev'
22
+ uses: TheDataEngineX/.github/.github/workflows/auto-pr-dev-to-main.yml@main
23
+ secrets: inherit
@@ -3,7 +3,7 @@ name: Notify Docs
3
3
  on:
4
4
  push:
5
5
  branches: [main]
6
- paths: ['docs/**', 'src/**/**.py']
6
+ paths: ['docs/**', 'src/**/*.py']
7
7
 
8
8
  jobs:
9
9
  notify:
@@ -0,0 +1,69 @@
1
+ name: Release
2
+
3
+ on:
4
+ push:
5
+ tags:
6
+ - 'v[0-9]+.[0-9]+.[0-9]+'
7
+
8
+ jobs:
9
+ build:
10
+ name: Build
11
+ runs-on: ubuntu-latest
12
+ permissions:
13
+ contents: read
14
+ steps:
15
+ - uses: actions/checkout@v6
16
+ - uses: astral-sh/setup-uv@v7
17
+ with:
18
+ version: "latest"
19
+ python-version: "3.13"
20
+ - run: uv build
21
+ - uses: actions/upload-artifact@v4
22
+ with:
23
+ name: dist
24
+ path: dist/
25
+
26
+ publish-pypi:
27
+ name: Publish to PyPI
28
+ needs: build
29
+ runs-on: ubuntu-latest
30
+ environment: ${{ vars.PYPI_ENVIRONMENT }}
31
+ permissions:
32
+ contents: read
33
+ id-token: write
34
+ steps:
35
+ - uses: actions/download-artifact@v4
36
+ with:
37
+ name: dist
38
+ path: dist/
39
+ - uses: pypa/gh-action-pypi-publish@release/v1
40
+
41
+ github-release:
42
+ name: GitHub Release + SBOM
43
+ needs: build
44
+ runs-on: ubuntu-latest
45
+ permissions:
46
+ contents: write
47
+ steps:
48
+ - uses: actions/checkout@v6
49
+ - uses: astral-sh/setup-uv@v7
50
+ with:
51
+ version: "latest"
52
+ - name: Extract version
53
+ id: version
54
+ run: |
55
+ VERSION=$(grep -m 1 "^version = " pyproject.toml | sed 's/version = "//;s/".*//')
56
+ echo "version=$VERSION" >> "$GITHUB_OUTPUT"
57
+ - name: Generate CycloneDX SBOM
58
+ run: |
59
+ uvx --from cyclonedx-bom cyclonedx-py environment \
60
+ -o sbom-dex-${{ steps.version.outputs.version }}.json \
61
+ --of json
62
+ - name: Create GitHub release + attach SBOM
63
+ env:
64
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
65
+ run: |
66
+ gh release create ${{ github.ref_name }} \
67
+ --title "DEX ${{ github.ref_name }}" \
68
+ --generate-notes \
69
+ "sbom-dex-${{ steps.version.outputs.version }}.json"
@@ -91,3 +91,6 @@ Notes.txt
91
91
  # AI Coding Assistant
92
92
  .continue/
93
93
  .playwright-mcp/
94
+
95
+ # Local task notes (tracked separately)
96
+ tasks/todo.md
@@ -2,7 +2,3 @@
2
2
  # https://github.com/gitleaks/gitleaks#configuration
3
3
 
4
4
  title = "DataEngineX Gitleaks Config"
5
-
6
- [allowlist]
7
- description = "Global allowlist"
8
- paths = []
@@ -4,12 +4,12 @@
4
4
 
5
5
  repos:
6
6
  - repo: https://github.com/gitleaks/gitleaks
7
- rev: v8.24.0
7
+ rev: v8.30.1
8
8
  hooks:
9
9
  - id: gitleaks
10
10
 
11
11
  - repo: https://github.com/pre-commit/pre-commit-hooks
12
- rev: v5.0.0
12
+ rev: v6.0.0
13
13
  hooks:
14
14
  - id: trailing-whitespace
15
15
  - id: end-of-file-fixer
@@ -28,23 +28,18 @@ repos:
28
28
  exclude: '^(site/|\.github/)'
29
29
 
30
30
  - repo: https://github.com/astral-sh/ruff-pre-commit
31
- rev: v0.15.0
31
+ rev: v0.15.12
32
32
  hooks:
33
33
  - id: ruff
34
34
  args: ["--fix"]
35
35
  - id: ruff-format
36
36
 
37
- - repo: https://github.com/pre-commit/mirrors-mypy
38
- rev: v1.16.1
37
+ - repo: local
39
38
  hooks:
40
39
  - id: mypy
41
- additional_dependencies:
42
- - fastapi>=0.128.4
43
- - httpx>=0.28.0
44
- - google-cloud-storage>=2.18.0
45
- - types-pyyaml
46
- - types-requests
40
+ name: mypy
41
+ entry: uv run mypy
42
+ language: system
47
43
  args: ["--config-file=pyproject.toml", "--no-incremental", "src/dataenginex"]
48
- language_version: python3.13
49
44
  pass_filenames: false
50
45
  files: ^src/dataenginex/
@@ -0,0 +1,3 @@
1
+ {
2
+ ".": "1.1.1"
3
+ }