dataenginex 0.6.0__tar.gz → 0.8.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dataenginex-0.8.0/.github/workflows/ci.yml +71 -0
- dataenginex-0.8.0/.github/workflows/claude.yml +49 -0
- dataenginex-0.8.0/.github/workflows/docker-build-push.yml +63 -0
- dataenginex-0.8.0/.github/workflows/enforce-dev-to-main.yml +20 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/.github/workflows/pypi-publish.yml +16 -24
- {dataenginex-0.6.0 → dataenginex-0.8.0}/.github/workflows/release-dataenginex.yml +14 -16
- {dataenginex-0.6.0 → dataenginex-0.8.0}/.github/workflows/security.yml +13 -29
- {dataenginex-0.6.0 → dataenginex-0.8.0}/.gitleaks.toml +1 -4
- {dataenginex-0.6.0 → dataenginex-0.8.0}/CHANGELOG.md +20 -3
- {dataenginex-0.6.0 → dataenginex-0.8.0}/CLAUDE.md +2 -2
- {dataenginex-0.6.0 → dataenginex-0.8.0}/Dockerfile +2 -2
- {dataenginex-0.6.0 → dataenginex-0.8.0}/PKG-INFO +18 -17
- {dataenginex-0.6.0 → dataenginex-0.8.0}/README.md +2 -2
- {dataenginex-0.6.0 → dataenginex-0.8.0}/docs/ARCHITECTURE.md +6 -8
- {dataenginex-0.6.0 → dataenginex-0.8.0}/docs/CI_CD.md +17 -17
- {dataenginex-0.6.0 → dataenginex-0.8.0}/docs/DEVELOPMENT.md +5 -8
- {dataenginex-0.6.0 → dataenginex-0.8.0}/docs/docs-hub.md +1 -1
- dataenginex-0.8.0/examples/05_rag_demo.py +112 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/examples/10_model_analysis.py +5 -3
- dataenginex-0.8.0/netlify.toml +11 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/poe_tasks.toml +16 -4
- {dataenginex-0.6.0 → dataenginex-0.8.0}/pyproject.toml +60 -36
- {dataenginex-0.6.0 → dataenginex-0.8.0}/src/dataenginex/RELEASE_NOTES.md +3 -3
- {dataenginex-0.6.0 → dataenginex-0.8.0}/src/dataenginex/__init__.py +1 -1
- {dataenginex-0.6.0 → dataenginex-0.8.0}/src/dataenginex/lakehouse/storage.py +23 -17
- {dataenginex-0.6.0 → dataenginex-0.8.0}/src/dataenginex/ml/__init__.py +4 -0
- dataenginex-0.8.0/src/dataenginex/ml/mlflow_registry.py +224 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/src/dataenginex/ml/vectorstore.py +71 -1
- dataenginex-0.8.0/src/dataenginex/secops/__init__.py +30 -0
- dataenginex-0.8.0/src/dataenginex/secops/audit.py +154 -0
- dataenginex-0.8.0/src/dataenginex/secops/gate.py +112 -0
- dataenginex-0.8.0/src/dataenginex/secops/masking.py +117 -0
- dataenginex-0.8.0/src/dataenginex/secops/pii.py +185 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/tests/unit/test_medallion.py +1 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/tests/unit/test_plugins.py +1 -1
- dataenginex-0.8.0/tests/unit/test_secops.py +300 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/uv.lock +1433 -919
- dataenginex-0.6.0/.github/workflows/ci.yml +0 -58
- dataenginex-0.6.0/examples/05_rag_demo.py +0 -85
- {dataenginex-0.6.0 → dataenginex-0.8.0}/.claude/commands/new-feature.md +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/.claude/commands/validate.md +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/.claude/settings.json +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/.env.template +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/.github/ISSUE_TEMPLATE/bug_report.yml +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/.github/ISSUE_TEMPLATE/config.yml +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/.github/ISSUE_TEMPLATE/feature_request.yml +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/.github/PULL_REQUEST_TEMPLATE.md +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/.github/dependabot.yml +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/.github/labels.yml +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/.github/workflows/label-sync.yml +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/.gitignore +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/.pre-commit-config.yaml +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/CODEOWNERS +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/CONTRIBUTING.md +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/LICENSE +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/docker-compose.test.yml +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/docs/CONTRIBUTING.md +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/docs/OBSERVABILITY.md +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/docs/RELEASE_NOTES.md +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/docs/SDLC.md +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/docs/SECURITY_SCANNING.md +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/docs/adr/0000-template.md +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/docs/adr/0001-medallion-architecture.md +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/docs/api-reference/api.md +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/docs/api-reference/core.md +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/docs/api-reference/dashboard.md +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/docs/api-reference/data.md +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/docs/api-reference/index.md +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/docs/api-reference/lakehouse.md +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/docs/api-reference/middleware.md +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/docs/api-reference/ml.md +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/docs/api-reference/plugins.md +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/docs/api-reference/warehouse.md +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/docs/index.md +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/docs/quickstart.md +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/docs/release-pr-template.md +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/docs/roadmap/project-roadmap.csv +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/docs/roadmap/project-roadmap.json +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/examples/01_hello_pipeline.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/examples/02_api_quickstart.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/examples/03_quality_gate.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/examples/04_ml_training.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/examples/06_llm_quickstart.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/examples/07_api_ingestion.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/examples/08_spark_ml.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/examples/09_feature_engineering.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/examples/GUIDE.md +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/examples/dashboard/dashboard_config.yaml +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/examples/dashboard/run_dashboard.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/mkdocs.yml +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/scripts/GUIDE.md +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/scripts/localstack/create-buckets.sh +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/scripts/promote.sh +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/sonar-project.properties +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/src/dataenginex/README.md +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/src/dataenginex/api/__init__.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/src/dataenginex/api/auth.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/src/dataenginex/api/errors.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/src/dataenginex/api/health.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/src/dataenginex/api/pagination.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/src/dataenginex/api/rate_limit.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/src/dataenginex/api/routers/__init__.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/src/dataenginex/core/__init__.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/src/dataenginex/core/medallion_architecture.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/src/dataenginex/core/quality.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/src/dataenginex/core/schemas.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/src/dataenginex/core/validators.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/src/dataenginex/dashboard/__init__.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/src/dataenginex/dashboard/app.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/src/dataenginex/dashboard/panels.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/src/dataenginex/data/__init__.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/src/dataenginex/data/connectors.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/src/dataenginex/data/profiler.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/src/dataenginex/data/registry.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/src/dataenginex/lakehouse/__init__.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/src/dataenginex/lakehouse/catalog.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/src/dataenginex/lakehouse/partitioning.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/src/dataenginex/middleware/__init__.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/src/dataenginex/middleware/logging_config.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/src/dataenginex/middleware/metrics.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/src/dataenginex/middleware/metrics_middleware.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/src/dataenginex/middleware/request_logging.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/src/dataenginex/middleware/tracing.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/src/dataenginex/ml/drift.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/src/dataenginex/ml/llm.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/src/dataenginex/ml/metrics.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/src/dataenginex/ml/registry.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/src/dataenginex/ml/scheduler.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/src/dataenginex/ml/serving.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/src/dataenginex/ml/training.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/src/dataenginex/plugins/__init__.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/src/dataenginex/plugins/registry.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/src/dataenginex/py.typed +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/src/dataenginex/warehouse/__init__.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/src/dataenginex/warehouse/lineage.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/src/dataenginex/warehouse/transforms.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/tasks/findings.md +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/tasks/lessons.md +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/tests/__init__.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/tests/conftest.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/tests/fixtures/__init__.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/tests/fixtures/sample_data.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/tests/fixtures/sample_jobs.csv +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/tests/fixtures/sample_jobs.json +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/tests/integration/__init__.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/tests/integration/test_storage_real.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/tests/load/__init__.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/tests/unit/__init__.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/tests/unit/test_data.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/tests/unit/test_drift_scheduler.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/tests/unit/test_errors.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/tests/unit/test_health.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/tests/unit/test_lakehouse.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/tests/unit/test_llm.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/tests/unit/test_logging.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/tests/unit/test_metrics.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/tests/unit/test_middleware.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/tests/unit/test_ml.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/tests/unit/test_spark_fixtures.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/tests/unit/test_storage_abstraction.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/tests/unit/test_tracing.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/tests/unit/test_vectorstore.py +0 -0
- {dataenginex-0.6.0 → dataenginex-0.8.0}/tests/unit/test_warehouse.py +0 -0
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main, dev]
|
|
6
|
+
pull_request:
|
|
7
|
+
branches: [main, dev]
|
|
8
|
+
|
|
9
|
+
permissions:
|
|
10
|
+
contents: read
|
|
11
|
+
|
|
12
|
+
jobs:
|
|
13
|
+
lint:
|
|
14
|
+
runs-on: ubuntu-latest
|
|
15
|
+
steps:
|
|
16
|
+
- uses: actions/checkout@v6
|
|
17
|
+
- uses: astral-sh/setup-uv@v7
|
|
18
|
+
with:
|
|
19
|
+
version: "latest"
|
|
20
|
+
python-version: "3.13"
|
|
21
|
+
- run: uv sync --all-extras
|
|
22
|
+
- run: uv run poe lint
|
|
23
|
+
- name: Check workspace.env version matches pyproject.toml
|
|
24
|
+
# workspace.env is gitignored — never present in CI. Use a shell -f check
|
|
25
|
+
# instead of hashFiles(), which is sandboxed to the checkout dir and
|
|
26
|
+
# cannot resolve ../ paths, causing the job to crash at evaluation time.
|
|
27
|
+
run: |
|
|
28
|
+
set -euo pipefail
|
|
29
|
+
WORKSPACE_ENV="../.github/workspace.env"
|
|
30
|
+
if [[ ! -f "$WORKSPACE_ENV" ]]; then
|
|
31
|
+
echo "workspace.env not found — skipping version drift check"
|
|
32
|
+
exit 0
|
|
33
|
+
fi
|
|
34
|
+
PYPROJECT_VERSION=$(python3 -c "import tomllib; t=tomllib.load(open('pyproject.toml','rb')); print(t['project']['version'])")
|
|
35
|
+
WORKSPACE_VERSION=$(grep '^VERSION_DEX=' "$WORKSPACE_ENV" | cut -d= -f2)
|
|
36
|
+
if [[ "$PYPROJECT_VERSION" != "$WORKSPACE_VERSION" ]]; then
|
|
37
|
+
echo "Version drift: pyproject.toml=$PYPROJECT_VERSION but workspace.env=$WORKSPACE_VERSION"
|
|
38
|
+
echo "Re-run .github/scripts/setup-workspace.sh to fix."
|
|
39
|
+
exit 1
|
|
40
|
+
fi
|
|
41
|
+
echo "Version check passed: $PYPROJECT_VERSION"
|
|
42
|
+
|
|
43
|
+
typecheck:
|
|
44
|
+
runs-on: ubuntu-latest
|
|
45
|
+
needs: lint
|
|
46
|
+
steps:
|
|
47
|
+
- uses: actions/checkout@v6
|
|
48
|
+
- uses: astral-sh/setup-uv@v7
|
|
49
|
+
with:
|
|
50
|
+
version: "latest"
|
|
51
|
+
python-version: "3.13"
|
|
52
|
+
- run: uv sync --all-extras
|
|
53
|
+
- run: uv run poe typecheck
|
|
54
|
+
|
|
55
|
+
test:
|
|
56
|
+
runs-on: ubuntu-latest
|
|
57
|
+
needs: [lint, typecheck]
|
|
58
|
+
steps:
|
|
59
|
+
- uses: actions/checkout@v6
|
|
60
|
+
- uses: astral-sh/setup-uv@v7
|
|
61
|
+
with:
|
|
62
|
+
version: "latest"
|
|
63
|
+
python-version: "3.13"
|
|
64
|
+
- run: uv sync --all-extras
|
|
65
|
+
- run: uv run pytest tests/ -x --tb=short -q --cov=dataenginex --cov-report=xml
|
|
66
|
+
- uses: codecov/codecov-action@v5
|
|
67
|
+
with:
|
|
68
|
+
flags: dataenginex
|
|
69
|
+
fail_ci_if_error: false
|
|
70
|
+
env:
|
|
71
|
+
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
name: Claude Code
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
issue_comment:
|
|
5
|
+
types: [created]
|
|
6
|
+
pull_request_review_comment:
|
|
7
|
+
types: [created]
|
|
8
|
+
issues:
|
|
9
|
+
types: [opened, assigned]
|
|
10
|
+
pull_request_review:
|
|
11
|
+
types: [submitted]
|
|
12
|
+
|
|
13
|
+
jobs:
|
|
14
|
+
claude:
|
|
15
|
+
if: |
|
|
16
|
+
(github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) ||
|
|
17
|
+
(github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) ||
|
|
18
|
+
(github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) ||
|
|
19
|
+
(github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')))
|
|
20
|
+
runs-on: ubuntu-latest
|
|
21
|
+
permissions:
|
|
22
|
+
contents: read
|
|
23
|
+
pull-requests: read
|
|
24
|
+
issues: read
|
|
25
|
+
id-token: write
|
|
26
|
+
actions: read # Required for Claude to read CI results on PRs
|
|
27
|
+
steps:
|
|
28
|
+
- name: Checkout repository
|
|
29
|
+
uses: actions/checkout@v4
|
|
30
|
+
with:
|
|
31
|
+
fetch-depth: 1
|
|
32
|
+
|
|
33
|
+
- name: Run Claude Code
|
|
34
|
+
id: claude
|
|
35
|
+
uses: anthropics/claude-code-action@v1
|
|
36
|
+
with:
|
|
37
|
+
claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
|
|
38
|
+
|
|
39
|
+
# This is an optional setting that allows Claude to read CI results on PRs
|
|
40
|
+
additional_permissions: |
|
|
41
|
+
actions: read
|
|
42
|
+
|
|
43
|
+
# Optional: Give a custom prompt to Claude. If this is not specified, Claude will perform the instructions specified in the comment that tagged it.
|
|
44
|
+
# prompt: 'Update the pull request description to include a summary of changes.'
|
|
45
|
+
|
|
46
|
+
# Optional: Add claude_args to customize behavior and configuration
|
|
47
|
+
# See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md
|
|
48
|
+
# or https://code.claude.com/docs/en/cli-reference for available options
|
|
49
|
+
# claude_args: '--allowed-tools Bash(gh pr:*)'
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
# Build and push Docker image to GitHub Container Registry (GHCR)
|
|
2
|
+
# Triggers on merge to main — publishes ghcr.io/thedataenginex/dex:<version>
|
|
3
|
+
|
|
4
|
+
name: Docker Build & Push
|
|
5
|
+
|
|
6
|
+
on:
|
|
7
|
+
push:
|
|
8
|
+
branches: [main]
|
|
9
|
+
paths:
|
|
10
|
+
- 'src/**'
|
|
11
|
+
- 'Dockerfile'
|
|
12
|
+
- 'pyproject.toml'
|
|
13
|
+
workflow_dispatch:
|
|
14
|
+
|
|
15
|
+
concurrency:
|
|
16
|
+
group: docker-${{ github.ref }}
|
|
17
|
+
cancel-in-progress: true
|
|
18
|
+
|
|
19
|
+
permissions:
|
|
20
|
+
contents: read
|
|
21
|
+
packages: write
|
|
22
|
+
|
|
23
|
+
jobs:
|
|
24
|
+
docker:
|
|
25
|
+
runs-on: ubuntu-latest
|
|
26
|
+
steps:
|
|
27
|
+
- uses: actions/checkout@v6
|
|
28
|
+
|
|
29
|
+
- name: Extract version from pyproject.toml
|
|
30
|
+
id: version
|
|
31
|
+
run: |
|
|
32
|
+
VERSION=$(python3 -c "import tomllib; t=tomllib.load(open('pyproject.toml','rb')); print(t['project']['version'])")
|
|
33
|
+
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
|
|
34
|
+
|
|
35
|
+
- name: Docker metadata
|
|
36
|
+
id: meta
|
|
37
|
+
uses: docker/metadata-action@v5
|
|
38
|
+
with:
|
|
39
|
+
images: ghcr.io/thedataenginex/dex
|
|
40
|
+
tags: |
|
|
41
|
+
type=raw,value=${{ steps.version.outputs.version }}
|
|
42
|
+
type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
|
|
43
|
+
type=sha,prefix=sha-
|
|
44
|
+
|
|
45
|
+
- name: Set up Docker Buildx
|
|
46
|
+
uses: docker/setup-buildx-action@v3
|
|
47
|
+
|
|
48
|
+
- name: Log in to GHCR
|
|
49
|
+
uses: docker/login-action@v3
|
|
50
|
+
with:
|
|
51
|
+
registry: ghcr.io
|
|
52
|
+
username: ${{ github.actor }}
|
|
53
|
+
password: ${{ secrets.GITHUB_TOKEN }}
|
|
54
|
+
|
|
55
|
+
- name: Build and push
|
|
56
|
+
uses: docker/build-push-action@v6
|
|
57
|
+
with:
|
|
58
|
+
context: .
|
|
59
|
+
push: true
|
|
60
|
+
tags: ${{ steps.meta.outputs.tags }}
|
|
61
|
+
labels: ${{ steps.meta.outputs.labels }}
|
|
62
|
+
cache-from: type=gha
|
|
63
|
+
cache-to: type=gha,mode=max
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
name: Enforce dev-to-main
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
pull_request:
|
|
5
|
+
branches:
|
|
6
|
+
- main
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
check-source-branch:
|
|
10
|
+
name: Verify PR source is dev
|
|
11
|
+
runs-on: ubuntu-latest
|
|
12
|
+
steps:
|
|
13
|
+
- name: Check source branch
|
|
14
|
+
run: |
|
|
15
|
+
if [ "${{ github.head_ref }}" != "dev" ]; then
|
|
16
|
+
echo "PRs to main must come from the 'dev' branch."
|
|
17
|
+
echo "Source branch: ${{ github.head_ref }}"
|
|
18
|
+
exit 1
|
|
19
|
+
fi
|
|
20
|
+
echo "Source branch is 'dev' — OK"
|
|
@@ -8,13 +8,6 @@ on:
|
|
|
8
8
|
release:
|
|
9
9
|
types:
|
|
10
10
|
- published
|
|
11
|
-
workflow_run:
|
|
12
|
-
workflows:
|
|
13
|
-
- Release DataEngineX
|
|
14
|
-
types:
|
|
15
|
-
- completed
|
|
16
|
-
branches:
|
|
17
|
-
- main
|
|
18
11
|
workflow_dispatch:
|
|
19
12
|
inputs:
|
|
20
13
|
tag:
|
|
@@ -32,7 +25,6 @@ jobs:
|
|
|
32
25
|
detect-dataenginex-changes:
|
|
33
26
|
name: Detect DataEngineX Changes
|
|
34
27
|
runs-on: ubuntu-latest
|
|
35
|
-
if: ${{ github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' }}
|
|
36
28
|
outputs:
|
|
37
29
|
dataenginex_changed: ${{ steps.detect.outputs.dataenginex_changed }}
|
|
38
30
|
previous_tag: ${{ steps.detect.outputs.previous_tag }}
|
|
@@ -42,7 +34,7 @@ jobs:
|
|
|
42
34
|
|
|
43
35
|
steps:
|
|
44
36
|
- name: Checkout code
|
|
45
|
-
uses: actions/checkout@
|
|
37
|
+
uses: actions/checkout@v6
|
|
46
38
|
with:
|
|
47
39
|
fetch-depth: 0
|
|
48
40
|
|
|
@@ -120,7 +112,7 @@ jobs:
|
|
|
120
112
|
|
|
121
113
|
steps:
|
|
122
114
|
- name: Notify Package Build Start
|
|
123
|
-
uses: actions/github-script@
|
|
115
|
+
uses: actions/github-script@v8
|
|
124
116
|
with:
|
|
125
117
|
script: |
|
|
126
118
|
if (!process.env.SLACK_WEBHOOK) return;
|
|
@@ -145,10 +137,10 @@ jobs:
|
|
|
145
137
|
TAG: ${{ needs.detect-dataenginex-changes.outputs.current_tag }}
|
|
146
138
|
|
|
147
139
|
- name: Checkout code
|
|
148
|
-
uses: actions/checkout@
|
|
140
|
+
uses: actions/checkout@v6
|
|
149
141
|
|
|
150
142
|
- name: Set up Python 3.12
|
|
151
|
-
uses: actions/setup-python@
|
|
143
|
+
uses: actions/setup-python@v6
|
|
152
144
|
with:
|
|
153
145
|
python-version: "3.12"
|
|
154
146
|
|
|
@@ -168,7 +160,7 @@ jobs:
|
|
|
168
160
|
uvx twine check dist/*
|
|
169
161
|
|
|
170
162
|
- name: Upload distributions
|
|
171
|
-
uses: actions/upload-artifact@
|
|
163
|
+
uses: actions/upload-artifact@v7
|
|
172
164
|
with:
|
|
173
165
|
name: python-distributions
|
|
174
166
|
path: dist/
|
|
@@ -177,14 +169,14 @@ jobs:
|
|
|
177
169
|
name: Publish to TestPyPI
|
|
178
170
|
runs-on: ubuntu-latest
|
|
179
171
|
needs: [detect-dataenginex-changes, build-distributions]
|
|
180
|
-
if: contains(fromJson('["release","
|
|
172
|
+
if: contains(fromJson('["release","workflow_dispatch"]'), github.event_name) && needs.detect-dataenginex-changes.outputs.dataenginex_changed == 'true' && vars.TESTPYPI_ENVIRONMENT != ''
|
|
181
173
|
environment: ${{ vars.TESTPYPI_ENVIRONMENT }}
|
|
182
174
|
permissions:
|
|
183
175
|
id-token: write
|
|
184
176
|
|
|
185
177
|
steps:
|
|
186
178
|
- name: Download distributions
|
|
187
|
-
uses: actions/download-artifact@
|
|
179
|
+
uses: actions/download-artifact@v8
|
|
188
180
|
with:
|
|
189
181
|
name: python-distributions
|
|
190
182
|
path: dist/
|
|
@@ -198,14 +190,14 @@ jobs:
|
|
|
198
190
|
name: Promote to PyPI
|
|
199
191
|
runs-on: ubuntu-latest
|
|
200
192
|
needs: [detect-dataenginex-changes, publish-testpypi]
|
|
201
|
-
if: contains(fromJson('["release","
|
|
193
|
+
if: contains(fromJson('["release","workflow_dispatch"]'), github.event_name) && needs.detect-dataenginex-changes.outputs.dataenginex_changed == 'true' && needs.detect-dataenginex-changes.outputs.approved_tag == 'true' && vars.PYPI_ENVIRONMENT != ''
|
|
202
194
|
environment: ${{ vars.PYPI_ENVIRONMENT }}
|
|
203
195
|
permissions:
|
|
204
196
|
id-token: write
|
|
205
197
|
|
|
206
198
|
steps:
|
|
207
199
|
- name: Download distributions
|
|
208
|
-
uses: actions/download-artifact@
|
|
200
|
+
uses: actions/download-artifact@v8
|
|
209
201
|
with:
|
|
210
202
|
name: python-distributions
|
|
211
203
|
path: dist/
|
|
@@ -215,7 +207,7 @@ jobs:
|
|
|
215
207
|
|
|
216
208
|
- name: Notify PyPI Publish Success
|
|
217
209
|
if: success()
|
|
218
|
-
uses: actions/github-script@
|
|
210
|
+
uses: actions/github-script@v8
|
|
219
211
|
with:
|
|
220
212
|
script: |
|
|
221
213
|
if (!process.env.SLACK_WEBHOOK) return;
|
|
@@ -244,11 +236,11 @@ jobs:
|
|
|
244
236
|
name: Skip PyPI Publish
|
|
245
237
|
runs-on: ubuntu-latest
|
|
246
238
|
needs: [detect-dataenginex-changes, publish-testpypi, publish-pypi]
|
|
247
|
-
if: always() && (!contains(fromJson('["release","
|
|
239
|
+
if: always() && (!contains(fromJson('["release","workflow_dispatch"]'), github.event_name) || needs.detect-dataenginex-changes.outputs.dataenginex_changed == 'false' || needs.detect-dataenginex-changes.outputs.approved_tag == 'false')
|
|
248
240
|
|
|
249
241
|
steps:
|
|
250
242
|
- name: Notify Publish Skipped
|
|
251
|
-
uses: actions/github-script@
|
|
243
|
+
uses: actions/github-script@v8
|
|
252
244
|
with:
|
|
253
245
|
script: |
|
|
254
246
|
if (!process.env.SLACK_WEBHOOK) return;
|
|
@@ -259,8 +251,8 @@ jobs:
|
|
|
259
251
|
const previousTag = process.env.PREVIOUS_TAG;
|
|
260
252
|
|
|
261
253
|
let reason = '';
|
|
262
|
-
if (eventName !== 'release' && eventName !== '
|
|
263
|
-
reason = 'Build validation only (no release/
|
|
254
|
+
if (eventName !== 'release' && eventName !== 'workflow_dispatch') {
|
|
255
|
+
reason = 'Build validation only (no release/workflow_dispatch event)';
|
|
264
256
|
} else if (dataenginexChanged === 'false') {
|
|
265
257
|
reason = `No changes in src/dataenginex/ since ${previousTag}`;
|
|
266
258
|
} else if (approvedTag === 'false') {
|
|
@@ -293,8 +285,8 @@ jobs:
|
|
|
293
285
|
|
|
294
286
|
- name: Skip publish
|
|
295
287
|
run: |
|
|
296
|
-
if [[ "${{ github.event_name }}" != "release" && "${{ github.event_name }}" != "
|
|
297
|
-
echo "Build validation only: publish jobs run only for release:published
|
|
288
|
+
if [[ "${{ github.event_name }}" != "release" && "${{ github.event_name }}" != "workflow_dispatch" ]]; then
|
|
289
|
+
echo "Build validation only: publish jobs run only for release:published or workflow_dispatch events."
|
|
298
290
|
elif [[ "${{ needs.detect-dataenginex-changes.outputs.dataenginex_changed }}" == "false" ]]; then
|
|
299
291
|
echo "Skipping TestPyPI/PyPI publish: no changes in src/dataenginex/ since previous tag ${{ needs.detect-dataenginex-changes.outputs.previous_tag }}."
|
|
300
292
|
elif [[ "${{ needs.detect-dataenginex-changes.outputs.approved_tag }}" == "false" ]]; then
|
|
@@ -8,8 +8,12 @@ on:
|
|
|
8
8
|
push:
|
|
9
9
|
branches:
|
|
10
10
|
- main
|
|
11
|
-
|
|
12
|
-
|
|
11
|
+
workflow_dispatch:
|
|
12
|
+
inputs:
|
|
13
|
+
version_override:
|
|
14
|
+
description: "Override version (leave empty to use pyproject.toml)"
|
|
15
|
+
required: false
|
|
16
|
+
type: string
|
|
13
17
|
|
|
14
18
|
permissions:
|
|
15
19
|
contents: write
|
|
@@ -26,7 +30,7 @@ jobs:
|
|
|
26
30
|
|
|
27
31
|
steps:
|
|
28
32
|
- name: Notify Release Start
|
|
29
|
-
uses: actions/github-script@
|
|
33
|
+
uses: actions/github-script@v8
|
|
30
34
|
if: always()
|
|
31
35
|
with:
|
|
32
36
|
script: |
|
|
@@ -51,7 +55,7 @@ jobs:
|
|
|
51
55
|
SLACK_WEBHOOK: ${{ env.SLACK_WEBHOOK }}
|
|
52
56
|
|
|
53
57
|
- name: Checkout code
|
|
54
|
-
uses: actions/checkout@
|
|
58
|
+
uses: actions/checkout@v6
|
|
55
59
|
with:
|
|
56
60
|
fetch-depth: 0
|
|
57
61
|
token: ${{ secrets.GITHUB_TOKEN }}
|
|
@@ -121,19 +125,13 @@ jobs:
|
|
|
121
125
|
--generate-notes \
|
|
122
126
|
--draft=false
|
|
123
127
|
|
|
124
|
-
- name: Set up Python for SBOM generation
|
|
125
|
-
if: steps.check_tag.outputs.tag_exists == 'false'
|
|
126
|
-
uses: actions/setup-python@v5
|
|
127
|
-
with:
|
|
128
|
-
python-version: "3.12"
|
|
129
|
-
|
|
130
128
|
- name: Generate CycloneDX SBOM
|
|
131
129
|
if: steps.check_tag.outputs.tag_exists == 'false'
|
|
132
130
|
run: |
|
|
133
|
-
|
|
134
|
-
cyclonedx-py environment \
|
|
135
|
-
|
|
136
|
-
--
|
|
131
|
+
# --output/-o and --of replace the old --output/--output-format flags (cyclonedx-bom 7+)
|
|
132
|
+
uvx --from cyclonedx-bom cyclonedx-py environment \
|
|
133
|
+
-o sbom-dataenginex-${{ steps.version.outputs.version }}.json \
|
|
134
|
+
--of json
|
|
137
135
|
echo "SBOM generated: sbom-dataenginex-${{ steps.version.outputs.version }}.json"
|
|
138
136
|
|
|
139
137
|
- name: Upload SBOM to release
|
|
@@ -147,7 +145,7 @@ jobs:
|
|
|
147
145
|
|
|
148
146
|
- name: Notify Release Success
|
|
149
147
|
if: success() && steps.check_tag.outputs.tag_exists == 'false'
|
|
150
|
-
uses: actions/github-script@
|
|
148
|
+
uses: actions/github-script@v8
|
|
151
149
|
with:
|
|
152
150
|
script: |
|
|
153
151
|
if (!process.env.SLACK_WEBHOOK) return;
|
|
@@ -175,7 +173,7 @@ jobs:
|
|
|
175
173
|
|
|
176
174
|
- name: Notify Release Failure
|
|
177
175
|
if: failure()
|
|
178
|
-
uses: actions/github-script@
|
|
176
|
+
uses: actions/github-script@v8
|
|
179
177
|
with:
|
|
180
178
|
script: |
|
|
181
179
|
if (!process.env.SLACK_WEBHOOK) return;
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# Security Scans
|
|
2
|
-
# Runs Trivy (misconfig + secret scan)
|
|
2
|
+
# Runs Trivy (misconfig + secret scan) on push/PR.
|
|
3
|
+
# CodeQL is handled by GitHub's default setup — do NOT add a custom codeql job
|
|
4
|
+
# (GitHub rejects SARIF uploads when default setup is enabled).
|
|
3
5
|
# Results appear in GitHub Security tab.
|
|
4
6
|
|
|
5
7
|
name: Security Scans
|
|
@@ -22,7 +24,7 @@ jobs:
|
|
|
22
24
|
runs-on: ubuntu-latest
|
|
23
25
|
steps:
|
|
24
26
|
- name: Notify Security Scan Start
|
|
25
|
-
uses: actions/github-script@
|
|
27
|
+
uses: actions/github-script@v8
|
|
26
28
|
if: always()
|
|
27
29
|
with:
|
|
28
30
|
script: |
|
|
@@ -37,7 +39,7 @@ jobs:
|
|
|
37
39
|
type: 'section',
|
|
38
40
|
text: {
|
|
39
41
|
type: 'mrkdwn',
|
|
40
|
-
text: `*🔒 Security Scans Started*\n*Branch:* ${process.env.REF}\n*Tools:* Trivy (repo)
|
|
42
|
+
text: `*🔒 Security Scans Started*\n*Branch:* ${process.env.REF}\n*Tools:* Trivy (repo)`
|
|
41
43
|
}
|
|
42
44
|
}]
|
|
43
45
|
})
|
|
@@ -47,7 +49,7 @@ jobs:
|
|
|
47
49
|
SLACK_WEBHOOK: ${{ env.SLACK_WEBHOOK }}
|
|
48
50
|
REF: ${{ github.ref_name }}
|
|
49
51
|
|
|
50
|
-
- uses: actions/checkout@
|
|
52
|
+
- uses: actions/checkout@v6
|
|
51
53
|
- name: Run Trivy repo security scan
|
|
52
54
|
uses: aquasecurity/trivy-action@0.35.0
|
|
53
55
|
with:
|
|
@@ -75,33 +77,16 @@ jobs:
|
|
|
75
77
|
severity: HIGH,CRITICAL
|
|
76
78
|
exit-code: '1'
|
|
77
79
|
|
|
78
|
-
codeql:
|
|
79
|
-
runs-on: ubuntu-latest
|
|
80
|
-
strategy:
|
|
81
|
-
fail-fast: false
|
|
82
|
-
matrix:
|
|
83
|
-
language: [ python, actions ]
|
|
84
|
-
steps:
|
|
85
|
-
- uses: actions/checkout@v4
|
|
86
|
-
- name: Initialize CodeQL
|
|
87
|
-
uses: github/codeql-action/init@v4
|
|
88
|
-
with:
|
|
89
|
-
languages: ${{ matrix.language }}
|
|
90
|
-
- name: Perform CodeQL Scan
|
|
91
|
-
uses: github/codeql-action/analyze@v4
|
|
92
|
-
with:
|
|
93
|
-
category: /language:${{ matrix.language }}
|
|
94
|
-
|
|
95
80
|
notify-security:
|
|
96
81
|
name: Notify Security Status
|
|
97
82
|
runs-on: ubuntu-latest
|
|
98
|
-
needs: [trivy-security
|
|
83
|
+
needs: [trivy-security]
|
|
99
84
|
if: always()
|
|
100
85
|
|
|
101
86
|
steps:
|
|
102
87
|
- name: Notify Security Success
|
|
103
|
-
if: ${{ needs.trivy-security.result == 'success'
|
|
104
|
-
uses: actions/github-script@
|
|
88
|
+
if: ${{ needs.trivy-security.result == 'success' }}
|
|
89
|
+
uses: actions/github-script@v8
|
|
105
90
|
with:
|
|
106
91
|
script: |
|
|
107
92
|
if (!process.env.SLACK_WEBHOOK) return;
|
|
@@ -123,8 +108,7 @@ jobs:
|
|
|
123
108
|
type: 'section',
|
|
124
109
|
fields: [
|
|
125
110
|
{ type: 'mrkdwn', text: `*Branch:*\n${process.env.REF}` },
|
|
126
|
-
{ type: 'mrkdwn', text: `*Trivy
|
|
127
|
-
{ type: 'mrkdwn', text: `*CodeQL:*\n✅ Passed` },
|
|
111
|
+
{ type: 'mrkdwn', text: `*Trivy:*\n✅ Passed` },
|
|
128
112
|
{ type: 'mrkdwn', text: `*Run:*\n<${process.env.URL}|View Details>` }
|
|
129
113
|
]
|
|
130
114
|
}
|
|
@@ -138,8 +122,8 @@ jobs:
|
|
|
138
122
|
URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
|
139
123
|
|
|
140
124
|
- name: Notify Security Issues
|
|
141
|
-
if: ${{ needs.trivy-security.result == 'failure'
|
|
142
|
-
uses: actions/github-script@
|
|
125
|
+
if: ${{ needs.trivy-security.result == 'failure' }}
|
|
126
|
+
uses: actions/github-script@v8
|
|
143
127
|
with:
|
|
144
128
|
script: |
|
|
145
129
|
if (!process.env.SLACK_WEBHOOK) return;
|
|
@@ -170,7 +154,7 @@ jobs:
|
|
|
170
154
|
},
|
|
171
155
|
{
|
|
172
156
|
type: 'context',
|
|
173
|
-
elements: [{ type: 'mrkdwn', text: `🔍 Trivy
|
|
157
|
+
elements: [{ type: 'mrkdwn', text: `🔍 Trivy found issues — review security tab for findings` }]
|
|
174
158
|
}
|
|
175
159
|
]
|
|
176
160
|
})
|
|
@@ -7,11 +7,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [0.7.1] - 2026-03-17
|
|
11
|
+
|
|
12
|
+
### Fixed
|
|
13
|
+
|
|
14
|
+
- **MLflow 3.x alias API** — `MLflowModelRegistry` now uses the alias-based API (`get_model_version_by_alias`, `set_registered_model_alias`, `delete_registered_model_alias`). MLflow 3.x removed all stage-based model management (`get_latest_versions`, `transition_model_version_stage`, `current_stage`).
|
|
15
|
+
- **Release workflow reliability** — removed `paths: - 'pyproject.toml'` filter from `release-dataenginex.yml`. GitHub suppresses all push-event workflow triggers when a commit modifies `.github/workflows/`; the tag-exists check inside the workflow handles no-ops.
|
|
16
|
+
- **Duplicate mypy override** — removed duplicate `[[tool.mypy.overrides]] module = ["mlflow.*"]` in `pyproject.toml` left by merge conflict.
|
|
17
|
+
|
|
18
|
+
### Changed
|
|
19
|
+
|
|
20
|
+
- **Cloud SDKs now optional** — `boto3`, `google-cloud-storage`, `google-cloud-bigquery` moved from core dependencies to `[project.optional-dependencies] cloud = [...]`. Install via `pip install dataenginex[cloud]`. Core install no longer requires any cloud SDK.
|
|
21
|
+
- **GCS emulator updated for 3.x** — `GCSStorage` now uses `ClientOptions(api_endpoint=...)` instead of the removed private `client._connection.API_BASE_URL`.
|
|
22
|
+
- **Dependency floors bumped** — pydantic 2.10, fastapi 0.135.1, pyarrow 23.0.1, sentence-transformers 5.3, mlflow 3.0, hatchling 1.29, mkdocstrings 1.0.
|
|
23
|
+
|
|
24
|
+
## [0.6.1] - 2026-03-15
|
|
25
|
+
|
|
10
26
|
### Added
|
|
11
27
|
|
|
12
|
-
-
|
|
13
|
-
-
|
|
14
|
-
- **
|
|
28
|
+
- **`SentenceTransformerEmbedder`** — thin wrapper over `sentence-transformers` (`all-MiniLM-L6-v2` default). Install via `uv add 'dataenginex[ml]'`. Implements the `embed_fn` protocol for `RAGPipeline`.
|
|
29
|
+
- **`RAGPipeline.answer(question, llm, ...)`** — full retrieve → augment → generate loop in one call. Combines `build_context` with any `LLMProvider.generate_with_context`.
|
|
30
|
+
- **GitHub Actions upgraded to Node.js 24** — `ci.yml`, `pypi-publish.yml`, `release-dataenginex.yml`, `security.yml` now use `actions/checkout@v6`, `actions/setup-python@v6`, `astral-sh/setup-uv@v7`.
|
|
31
|
+
- **`examples/05_rag_demo.py`** — end-to-end RAG demo with `--embed`, `--llm`, `--model` CLI flags; Ollama fallback to MockProvider; uses `RAGPipeline.answer()`.
|
|
15
32
|
|
|
16
33
|
## [0.6.0] - 2026-03-03
|
|
17
34
|
|
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
|
|
13
13
|
**Stack:** Python 3.12+ · FastAPI · uv · Ruff · mypy strict · pytest · Docker · Kubernetes (ArgoCD)
|
|
14
14
|
|
|
15
|
-
**Version:**
|
|
15
|
+
**Version:** `uv run poe version`
|
|
16
16
|
|
|
17
17
|
______________________________________________________________________
|
|
18
18
|
|
|
@@ -96,7 +96,7 @@ ______________________________________________________________________
|
|
|
96
96
|
|
|
97
97
|
| File | Purpose |
|
|
98
98
|
|------|---------|
|
|
99
|
-
| `pyproject.toml` | Package config (
|
|
99
|
+
| `pyproject.toml` | Package config (version source of truth) |
|
|
100
100
|
| `poe_tasks.toml` | All poe task definitions |
|
|
101
101
|
| `src/dataenginex/` | Framework source |
|
|
102
102
|
| `examples/` | Runnable examples (01–10) |
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
# ===============================================================
|
|
7
7
|
|
|
8
8
|
# --- Stage 1: Build dependencies ---
|
|
9
|
-
FROM python:3.
|
|
9
|
+
FROM python:3.13-slim AS builder
|
|
10
10
|
|
|
11
11
|
WORKDIR /build
|
|
12
12
|
|
|
@@ -29,7 +29,7 @@ ENV UV_PROJECT_ENVIRONMENT=/build/.venv \
|
|
|
29
29
|
RUN /root/.local/bin/uv sync --frozen --no-dev
|
|
30
30
|
|
|
31
31
|
# --- Stage 2: Minimal runtime image ---
|
|
32
|
-
FROM python:3.
|
|
32
|
+
FROM python:3.13-slim
|
|
33
33
|
|
|
34
34
|
WORKDIR /app
|
|
35
35
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dataenginex
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.8.0
|
|
4
4
|
Summary: DataEngineX - Core framework for data engineering projects
|
|
5
5
|
Author-email: Jay <jayapal.myaka99@gmail.com>
|
|
6
6
|
License: MIT License
|
|
@@ -25,26 +25,27 @@ License: MIT License
|
|
|
25
25
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
26
26
|
SOFTWARE.
|
|
27
27
|
License-File: LICENSE
|
|
28
|
-
Requires-Python: >=3.
|
|
29
|
-
Requires-Dist:
|
|
30
|
-
Requires-Dist:
|
|
31
|
-
Requires-Dist: fastapi>=0.128.4
|
|
32
|
-
Requires-Dist: google-cloud-bigquery>=3.30.0
|
|
33
|
-
Requires-Dist: google-cloud-storage>=2.18.0
|
|
28
|
+
Requires-Python: >=3.13
|
|
29
|
+
Requires-Dist: email-validator>=2.3.0
|
|
30
|
+
Requires-Dist: fastapi>=0.135.1
|
|
34
31
|
Requires-Dist: httpx>=0.28.0
|
|
35
32
|
Requires-Dist: loguru>=0.7.3
|
|
36
|
-
Requires-Dist: opentelemetry-api>=1.
|
|
37
|
-
Requires-Dist: opentelemetry-exporter-otlp>=1.
|
|
38
|
-
Requires-Dist: opentelemetry-instrumentation-fastapi>=0.
|
|
39
|
-
Requires-Dist: opentelemetry-sdk>=1.
|
|
40
|
-
Requires-Dist: prometheus-client>=0.24.
|
|
41
|
-
Requires-Dist: pyarrow>=
|
|
42
|
-
Requires-Dist: pydantic>=2.
|
|
43
|
-
Requires-Dist: python-dotenv>=1.2.
|
|
33
|
+
Requires-Dist: opentelemetry-api>=1.40.0
|
|
34
|
+
Requires-Dist: opentelemetry-exporter-otlp>=1.40.0
|
|
35
|
+
Requires-Dist: opentelemetry-instrumentation-fastapi>=0.61b0
|
|
36
|
+
Requires-Dist: opentelemetry-sdk>=1.40.0
|
|
37
|
+
Requires-Dist: prometheus-client>=0.24.1
|
|
38
|
+
Requires-Dist: pyarrow>=23.0.1
|
|
39
|
+
Requires-Dist: pydantic>=2.10.0
|
|
40
|
+
Requires-Dist: python-dotenv>=1.2.1
|
|
44
41
|
Requires-Dist: python-json-logger>=4.0.0
|
|
45
|
-
Requires-Dist: pyyaml>=6.0.
|
|
42
|
+
Requires-Dist: pyyaml>=6.0.3
|
|
46
43
|
Requires-Dist: structlog>=25.5.0
|
|
47
|
-
Requires-Dist: uvicorn>=0.
|
|
44
|
+
Requires-Dist: uvicorn>=0.42.0
|
|
45
|
+
Provides-Extra: cloud
|
|
46
|
+
Requires-Dist: boto3>=1.42.0; extra == 'cloud'
|
|
47
|
+
Requires-Dist: google-cloud-bigquery>=3.40.0; extra == 'cloud'
|
|
48
|
+
Requires-Dist: google-cloud-storage>=3.0.0; extra == 'cloud'
|
|
48
49
|
Description-Content-Type: text/markdown
|
|
49
50
|
|
|
50
51
|
# dataenginex
|
|
@@ -62,7 +62,7 @@ DEX/
|
|
|
62
62
|
│
|
|
63
63
|
├── Dockerfile # Multi-stage, non-root, port 8000
|
|
64
64
|
├── docker-compose.test.yml # S3 + GCS emulators for integration tests
|
|
65
|
-
├── pyproject.toml # Package config
|
|
65
|
+
├── pyproject.toml # Package config
|
|
66
66
|
└── poe_tasks.toml # Task runner (poe)
|
|
67
67
|
```
|
|
68
68
|
|
|
@@ -205,4 +205,4 @@ The project identity is protected — see the [org Trademark Policy](https://git
|
|
|
205
205
|
|
|
206
206
|
______________________________________________________________________
|
|
207
207
|
|
|
208
|
-
**Version**:
|
|
208
|
+
**Version**: [](https://pypi.org/project/dataenginex/) | **License**: MIT | **Python**: 3.12+
|