ray-doris 1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. ray_doris-1.0/.dockerignore +10 -0
  2. ray_doris-1.0/.github/workflows/ci.yml +163 -0
  3. ray_doris-1.0/.github/workflows/release.yml +333 -0
  4. ray_doris-1.0/.github/workflows/slow-integration.yml +90 -0
  5. ray_doris-1.0/.gitignore +15 -0
  6. ray_doris-1.0/.pre-commit-config.yaml +14 -0
  7. ray_doris-1.0/.readthedocs.yaml +23 -0
  8. ray_doris-1.0/CHANGELOG.md +28 -0
  9. ray_doris-1.0/CONTRIBUTING.md +139 -0
  10. ray_doris-1.0/LICENSE +186 -0
  11. ray_doris-1.0/NOTICE +4 -0
  12. ray_doris-1.0/PKG-INFO +450 -0
  13. ray_doris-1.0/README.md +410 -0
  14. ray_doris-1.0/SECURITY.md +24 -0
  15. ray_doris-1.0/doc/Makefile +31 -0
  16. ray_doris-1.0/doc/requirements-doc.lock.txt +158 -0
  17. ray_doris-1.0/doc/requirements-doc.txt +12 -0
  18. ray_doris-1.0/doc/source/_static/custom.css +15 -0
  19. ray_doris-1.0/doc/source/_templates/main-sidebar.html +15 -0
  20. ray_doris-1.0/doc/source/api/api.md +122 -0
  21. ray_doris-1.0/doc/source/architecture.md +119 -0
  22. ray_doris-1.0/doc/source/compatibility.md +94 -0
  23. ray_doris-1.0/doc/source/conf.py +109 -0
  24. ray_doris-1.0/doc/source/contributing/index.md +76 -0
  25. ray_doris-1.0/doc/source/examples/index.md +122 -0
  26. ray_doris-1.0/doc/source/faq.md +85 -0
  27. ray_doris-1.0/doc/source/index.md +113 -0
  28. ray_doris-1.0/doc/source/key-concepts.md +85 -0
  29. ray_doris-1.0/doc/source/quickstart.md +80 -0
  30. ray_doris-1.0/doc/source/spelling_wordlist.txt +70 -0
  31. ray_doris-1.0/doc/source/user-guide/configure-transports.md +109 -0
  32. ray_doris-1.0/doc/source/user-guide/index.md +76 -0
  33. ray_doris-1.0/doc/source/user-guide/read-data.md +136 -0
  34. ray_doris-1.0/doc/source/user-guide/secure-connections.md +151 -0
  35. ray_doris-1.0/doc/source/user-guide/troubleshooting.md +102 -0
  36. ray_doris-1.0/doc/source/user-guide/tune-parallelism.md +93 -0
  37. ray_doris-1.0/doc/source/user-guide/work-with-schemas.md +75 -0
  38. ray_doris-1.0/doc/source/user-guide/write-data.md +109 -0
  39. ray_doris-1.0/examples/quickstart.py +21 -0
  40. ray_doris-1.0/pyproject.toml +82 -0
  41. ray_doris-1.0/release-notes/v1.0.md +27 -0
  42. ray_doris-1.0/src/ray_doris/__init__.py +51 -0
  43. ray_doris-1.0/src/ray_doris/_api.py +73 -0
  44. ray_doris-1.0/src/ray_doris/_compat.py +117 -0
  45. ray_doris-1.0/src/ray_doris/_errors.py +49 -0
  46. ray_doris-1.0/src/ray_doris/_models.py +292 -0
  47. ray_doris-1.0/src/ray_doris/_planner.py +401 -0
  48. ray_doris-1.0/src/ray_doris/_readers.py +396 -0
  49. ray_doris-1.0/src/ray_doris/_schema.py +152 -0
  50. ray_doris-1.0/src/ray_doris/_sql.py +116 -0
  51. ray_doris-1.0/src/ray_doris/datasource.py +161 -0
  52. ray_doris-1.0/src/ray_doris/py.typed +0 -0
  53. ray_doris-1.0/src/ray_doris/write/__init__.py +15 -0
  54. ray_doris-1.0/src/ray_doris/write/api.py +51 -0
  55. ray_doris-1.0/src/ray_doris/write/connection.py +201 -0
  56. ray_doris-1.0/src/ray_doris/write/datasink.py +242 -0
  57. ray_doris-1.0/src/ray_doris/write/metadata.py +367 -0
  58. ray_doris-1.0/src/ray_doris/write/options.py +148 -0
  59. ray_doris-1.0/src/ray_doris/write/serialization.py +112 -0
  60. ray_doris-1.0/src/ray_doris/write/stream_load.py +396 -0
  61. ray_doris-1.0/tests/__init__.py +1 -0
  62. ray_doris-1.0/tests/contract/__init__.py +1 -0
  63. ray_doris-1.0/tests/contract/test_write_datasink.py +112 -0
  64. ray_doris-1.0/tests/integration/conftest.py +355 -0
  65. ray_doris-1.0/tests/integration/docker/be.Dockerfile +3 -0
  66. ray_doris-1.0/tests/integration/docker/be.conf +9 -0
  67. ray_doris-1.0/tests/integration/docker/fe.Dockerfile +3 -0
  68. ray_doris-1.0/tests/integration/docker/fe.conf +10 -0
  69. ray_doris-1.0/tests/integration/docker-compose.yml +58 -0
  70. ray_doris-1.0/tests/integration/test_distributed_read.py +16 -0
  71. ray_doris-1.0/tests/integration/test_flight_read.py +69 -0
  72. ray_doris-1.0/tests/integration/test_mysql_read.py +222 -0
  73. ray_doris-1.0/tests/integration/test_query_plan_errors.py +49 -0
  74. ray_doris-1.0/tests/integration/test_write.py +193 -0
  75. ray_doris-1.0/tests/it-ledger.md +44 -0
  76. ray_doris-1.0/tests/slow_integration/_cluster.py +419 -0
  77. ray_doris-1.0/tests/slow_integration/conftest.py +46 -0
  78. ray_doris-1.0/tests/slow_integration/docker/be.Dockerfile +3 -0
  79. ray_doris-1.0/tests/slow_integration/docker/be.conf +9 -0
  80. ray_doris-1.0/tests/slow_integration/docker/fe.Dockerfile +3 -0
  81. ray_doris-1.0/tests/slow_integration/docker/fe.conf +14 -0
  82. ray_doris-1.0/tests/slow_integration/docker/generate-tls.sh +69 -0
  83. ray_doris-1.0/tests/slow_integration/docker/haproxy.cfg +78 -0
  84. ray_doris-1.0/tests/slow_integration/docker/ray.Dockerfile +20 -0
  85. ray_doris-1.0/tests/slow_integration/docker/write_fault_server.py +61 -0
  86. ray_doris-1.0/tests/slow_integration/docker-compose.yml +283 -0
  87. ray_doris-1.0/tests/slow_integration/run.sh +130 -0
  88. ray_doris-1.0/tests/slow_integration/test_distributed_cluster.py +652 -0
  89. ray_doris-1.0/tests/slow_integration/write_result.py +50 -0
  90. ray_doris-1.0/tests/unit/test_api.py +59 -0
  91. ray_doris-1.0/tests/unit/test_compat.py +106 -0
  92. ray_doris-1.0/tests/unit/test_datasource.py +126 -0
  93. ray_doris-1.0/tests/unit/test_models.py +193 -0
  94. ray_doris-1.0/tests/unit/test_planner.py +523 -0
  95. ray_doris-1.0/tests/unit/test_readers.py +673 -0
  96. ray_doris-1.0/tests/unit/test_release.py +200 -0
  97. ray_doris-1.0/tests/unit/test_schema.py +82 -0
  98. ray_doris-1.0/tests/unit/test_sql.py +64 -0
  99. ray_doris-1.0/tests/unit/test_write.py +536 -0
  100. ray_doris-1.0/tools/check_docs.py +196 -0
  101. ray_doris-1.0/tools/check_release.py +179 -0
  102. ray_doris-1.0/tools/check_release_workflow.py +154 -0
  103. ray_doris-1.0/tools/check_slow_result.py +101 -0
  104. ray_doris-1.0/tools/check_workflow_pins.py +48 -0
  105. ray_doris-1.0/tools/verify_release_artifacts.py +91 -0
  106. ray_doris-1.0/tools/verify_release_tag.py +43 -0
@@ -0,0 +1,10 @@
1
+ **
2
+ !pyproject.toml
3
+ !README.md
4
+ !LICENSE
5
+ !NOTICE
6
+ !src/
7
+ !src/**
8
+ !tests/
9
+ !tests/slow_integration/
10
+ !tests/slow_integration/**
@@ -0,0 +1,163 @@
1
+ name: CI
2
+
3
+ on:
4
+ workflow_call:
5
+ inputs:
6
+ candidate_sha:
7
+ description: Immutable release candidate commit to test
8
+ required: false
9
+ type: string
10
+ pull_request:
11
+ push:
12
+ branches: [master]
13
+ schedule:
14
+ - cron: "17 3 * * 1"
15
+
16
+ concurrency:
17
+ group: ci-${{ github.workflow }}-${{ github.ref }}
18
+ cancel-in-progress: true
19
+
20
+ jobs:
21
+ unit:
22
+ name: Python ${{ matrix.python }} / Ray ${{ matrix.ray }}
23
+ runs-on: ubuntu-latest
24
+ timeout-minutes: 15
25
+ strategy:
26
+ fail-fast: false
27
+ matrix:
28
+ include:
29
+ - python: "3.9"
30
+ ray: "2.49.2"
31
+ - python: "3.10"
32
+ ray: "2.55.1"
33
+ - python: "3.12"
34
+ ray: "2.56.1"
35
+ steps:
36
+ - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
37
+ with:
38
+ persist-credentials: false
39
+ ref: ${{ inputs.candidate_sha || github.sha }}
40
+ - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0
41
+ with:
42
+ python-version: ${{ matrix.python }}
43
+ - run: uv venv --python ${{ matrix.python }}
44
+ - run: uv pip install -e ".[test]" "ray[data]==${{ matrix.ray }}"
45
+ - run: .venv/bin/python -m pytest tests/unit tests/contract --cov=ray_doris --cov-report=term-missing
46
+
47
+ quality:
48
+ name: Static quality
49
+ runs-on: ubuntu-latest
50
+ timeout-minutes: 10
51
+ steps:
52
+ - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
53
+ with:
54
+ persist-credentials: false
55
+ ref: ${{ inputs.candidate_sha || github.sha }}
56
+ - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0
57
+ with:
58
+ python-version: "3.12"
59
+ - run: uv venv --python 3.12
60
+ - run: uv pip install -e ".[dev,flight]" "ray[data]==2.56.1"
61
+ - run: .venv/bin/ruff format --check .
62
+ - run: .venv/bin/ruff check .
63
+ - run: .venv/bin/mypy
64
+ - run: .venv/bin/pre-commit run --all-files
65
+ - run: >-
66
+ .venv/bin/python tools/check_workflow_pins.py
67
+ .github/workflows/ci.yml
68
+ .github/workflows/release.yml
69
+ .github/workflows/slow-integration.yml
70
+ - run: >-
71
+ .venv/bin/python tools/check_release_workflow.py
72
+ .github/workflows/ci.yml .github/workflows/release.yml
73
+
74
+ docs:
75
+ name: Documentation
76
+ runs-on: ubuntu-latest
77
+ timeout-minutes: 15
78
+ steps:
79
+ - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
80
+ with:
81
+ persist-credentials: false
82
+ ref: ${{ inputs.candidate_sha || github.sha }}
83
+ - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0
84
+ with:
85
+ python-version: "3.12"
86
+ - name: Install spelling system dependency
87
+ run: |
88
+ sudo apt-get update
89
+ sudo apt-get install --yes enchant-2
90
+ - name: Install documentation environment
91
+ run: |
92
+ uv venv .docs-venv --python 3.12
93
+ uv pip install \
94
+ --python .docs-venv/bin/python \
95
+ --requirement doc/requirements-doc.lock.txt
96
+ uv pip install \
97
+ --python .docs-venv/bin/python \
98
+ --no-build-isolation \
99
+ --no-deps \
100
+ --editable .
101
+ - name: Check documentation contracts
102
+ run: .docs-venv/bin/python tools/check_docs.py
103
+ - name: Build strict documentation
104
+ run: make -C doc strict SPHINXBUILD=../.docs-venv/bin/sphinx-build
105
+ - name: Check documentation spelling
106
+ run: make -C doc spelling SPHINXBUILD=../.docs-venv/bin/sphinx-build
107
+ - uses: lycheeverse/lychee-action@e7477775783ea5526144ba13e8db5eec57747ce8 # v2.4.1
108
+ with:
109
+ args: >-
110
+ --no-progress
111
+ --max-retries 3
112
+ README.md CONTRIBUTING.md SECURITY.md CHANGELOG.md doc/source
113
+
114
+ integration:
115
+ name: Doris 4.0.6 integration
116
+ runs-on: ubuntu-latest
117
+ timeout-minutes: 25
118
+ steps:
119
+ - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
120
+ with:
121
+ persist-credentials: false
122
+ ref: ${{ inputs.candidate_sha || github.sha }}
123
+ - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0
124
+ with:
125
+ python-version: "3.12"
126
+ - run: uv venv --python 3.12
127
+ - run: uv pip install -e ".[test,flight]" "ray[data]==2.56.1"
128
+ - run: docker compose -f tests/integration/docker-compose.yml up -d --build
129
+ - run: .venv/bin/python -m pytest tests/integration -v
130
+ - name: Capture Doris logs
131
+ if: always()
132
+ run: docker compose -f tests/integration/docker-compose.yml logs --no-color > doris-it.log
133
+ - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
134
+ if: always()
135
+ with:
136
+ name: doris-it-log
137
+ path: doris-it.log
138
+ - name: Stop Doris
139
+ if: always()
140
+ run: docker compose -f tests/integration/docker-compose.yml down -v --rmi local
141
+
142
+ package:
143
+ name: Package
144
+ runs-on: ubuntu-latest
145
+ timeout-minutes: 15
146
+ steps:
147
+ - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
148
+ with:
149
+ persist-credentials: false
150
+ ref: ${{ inputs.candidate_sha || github.sha }}
151
+ - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0
152
+ with:
153
+ python-version: "3.12"
154
+ - run: uv venv --python 3.12
155
+ - run: uv pip install build twine
156
+ - run: .venv/bin/python -m build
157
+ - run: .venv/bin/twine check dist/*
158
+ - run: uv venv --python 3.12 .venv-package
159
+ - run: uv pip install --python .venv-package/bin/python dist/*.whl
160
+ - run: .venv-package/bin/python -c "import ray_doris; print(ray_doris.__version__)"
161
+ - run: uv venv --python 3.12 .venv-sdist
162
+ - run: uv pip install --python .venv-sdist/bin/python dist/*.tar.gz
163
+ - run: .venv-sdist/bin/python -c "import ray_doris; print(ray_doris.__version__)"
@@ -0,0 +1,333 @@
1
+ name: Release
2
+
3
+ on:
4
+ push:
5
+ tags: ["v*"]
6
+ workflow_dispatch:
7
+ inputs:
8
+ candidate_sha:
9
+ description: Full master-reachable commit SHA to validate without publishing
10
+ required: true
11
+ type: string
12
+ expected_version:
13
+ description: Exact package version expected at the candidate commit
14
+ required: true
15
+ type: string
16
+
17
+ permissions:
18
+ contents: read
19
+ actions: read
20
+
21
+ concurrency:
22
+ group: release-${{ github.event_name == 'push' && github.ref || inputs.candidate_sha }}
23
+ cancel-in-progress: false
24
+
25
+ jobs:
26
+ candidate:
27
+ name: Validate release candidate
28
+ runs-on: ubuntu-latest
29
+ outputs:
30
+ mode: ${{ steps.candidate.outputs.mode }}
31
+ sha: ${{ steps.candidate.outputs.sha }}
32
+ version: ${{ steps.candidate.outputs.version }}
33
+ tag: ${{ steps.candidate.outputs.tag }}
34
+ master_sha: ${{ steps.candidate.outputs.master_sha }}
35
+ steps:
36
+ - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
37
+ with:
38
+ fetch-depth: 0
39
+ persist-credentials: false
40
+ ref: master
41
+ - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
42
+ with:
43
+ python-version: "3.12"
44
+ - name: Fetch protected master and tag identities
45
+ run: >-
46
+ git fetch --force origin
47
+ +refs/heads/master:refs/remotes/origin/master
48
+ +refs/tags/*:refs/tags/*
49
+ - name: Resolve immutable candidate
50
+ id: candidate
51
+ env:
52
+ CANDIDATE_REF: ${{ github.event_name == 'workflow_dispatch' && inputs.candidate_sha || github.ref }}
53
+ EXPECTED_VERSION: ${{ github.event_name == 'workflow_dispatch' && inputs.expected_version || '' }}
54
+ MODE: ${{ github.event_name == 'workflow_dispatch' && 'dry-run' || 'tag' }}
55
+ RELEASE_TAG: ${{ github.event_name == 'push' && github.ref_name || '' }}
56
+ EVENT_SHA: ${{ github.event_name == 'push' && github.event.after || '' }}
57
+ EVENT_CREATED: ${{ github.event_name == 'push' && github.event.created || false }}
58
+ EVENT_DELETED: ${{ github.event_name == 'push' && github.event.deleted || false }}
59
+ EVENT_FORCED: ${{ github.event_name == 'push' && github.event.forced || false }}
60
+ run: >-
61
+ python tools/check_release.py
62
+ --mode "${MODE}"
63
+ --candidate-ref "${CANDIDATE_REF}"
64
+ --expected-version "${EXPECTED_VERSION}"
65
+ --tag "${RELEASE_TAG}"
66
+ --master-ref origin/master
67
+ --event-sha "${EVENT_SHA}"
68
+ --event-created "${EVENT_CREATED}"
69
+ --event-deleted "${EVENT_DELETED}"
70
+ --event-forced "${EVENT_FORCED}"
71
+ --github-output "${GITHUB_OUTPUT}"
72
+
73
+ verify:
74
+ name: Candidate CI gates
75
+ needs: candidate
76
+ uses: ./.github/workflows/ci.yml
77
+ with:
78
+ candidate_sha: ${{ needs.candidate.outputs.sha }}
79
+
80
+ build:
81
+ name: Build and verify distributions
82
+ needs: [candidate, verify]
83
+ runs-on: ubuntu-latest
84
+ outputs:
85
+ distributions_artifact_id: ${{ steps.upload.outputs.artifact-id }}
86
+ steps:
87
+ - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
88
+ with:
89
+ persist-credentials: false
90
+ ref: ${{ needs.candidate.outputs.sha }}
91
+ - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0
92
+ with:
93
+ enable-cache: true
94
+ python-version: "3.12"
95
+ - run: uv venv --python 3.12
96
+ - run: uv pip install --python .venv/bin/python "build==1.3.0" "twine==6.2.0"
97
+ - run: .venv/bin/python -m build --outdir release-artifacts/packages
98
+ - run: .venv/bin/twine check release-artifacts/packages/*
99
+ - run: >-
100
+ .venv/bin/python tools/verify_release_artifacts.py release-artifacts/packages
101
+ --name ray-doris
102
+ --version "${{ needs.candidate.outputs.version }}"
103
+ --manifest release-artifacts/SHA256SUMS
104
+ --prepare
105
+ - name: Upload immutable distributions
106
+ id: upload
107
+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
108
+ with:
109
+ name: distributions
110
+ path: release-artifacts/
111
+ if-no-files-found: error
112
+
113
+ install-smoke:
114
+ name: Install ${{ matrix.distribution }} ${{ matrix.profile }}
115
+ needs: [candidate, verify, build]
116
+ runs-on: ubuntu-latest
117
+ strategy:
118
+ fail-fast: false
119
+ matrix:
120
+ include:
121
+ - profile: base
122
+ extra: ""
123
+ distribution: wheel
124
+ - profile: flight
125
+ extra: "[flight]"
126
+ distribution: wheel
127
+ - profile: base
128
+ extra: ""
129
+ distribution: sdist
130
+ - profile: flight
131
+ extra: "[flight]"
132
+ distribution: sdist
133
+ steps:
134
+ - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
135
+ with:
136
+ persist-credentials: false
137
+ ref: ${{ needs.candidate.outputs.sha }}
138
+ - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
139
+ with:
140
+ python-version: "3.12"
141
+ - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
142
+ with:
143
+ artifact-ids: ${{ needs.build.outputs.distributions_artifact_id }}
144
+ path: release/
145
+ merge-multiple: true
146
+ - run: >-
147
+ python tools/verify_release_artifacts.py release/packages
148
+ --name ray-doris
149
+ --version "${{ needs.candidate.outputs.version }}"
150
+ --manifest release/SHA256SUMS
151
+ - name: Install built distribution
152
+ env:
153
+ EXTRA: ${{ matrix.extra }}
154
+ DISTRIBUTION: ${{ matrix.distribution }}
155
+ run: |
156
+ if [ "$DISTRIBUTION" = "wheel" ]; then
157
+ package="$(find release/packages -maxdepth 1 -type f -name '*.whl')"
158
+ else
159
+ package="$(find release/packages -maxdepth 1 -type f -name '*.tar.gz')"
160
+ fi
161
+ test -n "$package"
162
+ python -m pip install "ray-doris${EXTRA} @ file://${GITHUB_WORKSPACE}/${package}"
163
+ - run: python -c "import ray_doris; print(ray_doris.__version__)"
164
+
165
+ testpypi-publish:
166
+ name: Publish formal artifact to TestPyPI
167
+ if: >-
168
+ always() &&
169
+ github.event_name == 'push' &&
170
+ startsWith(github.ref, 'refs/tags/v') &&
171
+ needs.candidate.result == 'success' &&
172
+ needs.verify.result == 'success' &&
173
+ needs.build.result == 'success' &&
174
+ needs.install-smoke.result == 'success' &&
175
+ needs.candidate.outputs.mode == 'tag'
176
+ needs: [candidate, verify, build, install-smoke]
177
+ runs-on: ubuntu-latest
178
+ environment: testpypi
179
+ permissions:
180
+ contents: read
181
+ id-token: write
182
+ steps:
183
+ - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
184
+ with:
185
+ persist-credentials: false
186
+ ref: ${{ needs.candidate.outputs.sha }}
187
+ - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
188
+ with:
189
+ python-version: "3.12"
190
+ - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
191
+ with:
192
+ artifact-ids: ${{ needs.build.outputs.distributions_artifact_id }}
193
+ path: release/
194
+ merge-multiple: true
195
+ - run: >-
196
+ python tools/verify_release_tag.py
197
+ --tag "${{ needs.candidate.outputs.tag }}"
198
+ --expected-sha "${{ needs.candidate.outputs.sha }}"
199
+ - run: >-
200
+ python tools/verify_release_artifacts.py release/packages
201
+ --name ray-doris
202
+ --version "${{ needs.candidate.outputs.version }}"
203
+ --manifest release/SHA256SUMS
204
+ - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2
205
+ with:
206
+ packages-dir: release/packages/
207
+ repository-url: https://test.pypi.org/legacy/
208
+
209
+ testpypi-smoke:
210
+ name: TestPyPI smoke (${{ matrix.profile }})
211
+ if: >-
212
+ always() &&
213
+ github.event_name == 'push' &&
214
+ startsWith(github.ref, 'refs/tags/v') &&
215
+ needs.candidate.result == 'success' &&
216
+ needs.testpypi-publish.result == 'success' &&
217
+ needs.candidate.outputs.mode == 'tag'
218
+ needs: [candidate, testpypi-publish]
219
+ runs-on: ubuntu-latest
220
+ strategy:
221
+ fail-fast: false
222
+ matrix:
223
+ include:
224
+ - profile: base
225
+ extra: ""
226
+ - profile: flight
227
+ extra: "[flight]"
228
+ steps:
229
+ - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
230
+ with:
231
+ python-version: "3.12"
232
+ - name: Install from TestPyPI with PyPI dependency fallback
233
+ env:
234
+ EXTRA: ${{ matrix.extra }}
235
+ VERSION: ${{ needs.candidate.outputs.version }}
236
+ run: >-
237
+ python -m pip install
238
+ --index-url https://test.pypi.org/simple/
239
+ --extra-index-url https://pypi.org/simple/
240
+ "ray-doris${EXTRA}==${VERSION}"
241
+ - run: python -c "import ray_doris; print(ray_doris.__version__)"
242
+
243
+ publish:
244
+ name: Publish to PyPI
245
+ if: >-
246
+ always() &&
247
+ github.event_name == 'push' &&
248
+ startsWith(github.ref, 'refs/tags/v') &&
249
+ needs.candidate.result == 'success' &&
250
+ needs.verify.result == 'success' &&
251
+ needs.build.result == 'success' &&
252
+ needs.install-smoke.result == 'success' &&
253
+ needs.testpypi-publish.result == 'success' &&
254
+ needs.testpypi-smoke.result == 'success' &&
255
+ needs.candidate.outputs.mode == 'tag'
256
+ needs: [candidate, verify, build, install-smoke, testpypi-publish, testpypi-smoke]
257
+ runs-on: ubuntu-latest
258
+ environment: pypi
259
+ permissions:
260
+ contents: read
261
+ id-token: write
262
+ steps:
263
+ - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
264
+ with:
265
+ persist-credentials: false
266
+ ref: ${{ needs.candidate.outputs.sha }}
267
+ - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
268
+ with:
269
+ python-version: "3.12"
270
+ - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
271
+ with:
272
+ artifact-ids: ${{ needs.build.outputs.distributions_artifact_id }}
273
+ path: release/
274
+ merge-multiple: true
275
+ - run: >-
276
+ python tools/verify_release_tag.py
277
+ --tag "${{ needs.candidate.outputs.tag }}"
278
+ --expected-sha "${{ needs.candidate.outputs.sha }}"
279
+ - run: >-
280
+ python tools/verify_release_artifacts.py release/packages
281
+ --name ray-doris
282
+ --version "${{ needs.candidate.outputs.version }}"
283
+ --manifest release/SHA256SUMS
284
+ - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2
285
+ with:
286
+ packages-dir: release/packages/
287
+
288
+ github-release:
289
+ name: Create GitHub Release
290
+ if: >-
291
+ always() &&
292
+ github.event_name == 'push' &&
293
+ startsWith(github.ref, 'refs/tags/v') &&
294
+ needs.candidate.result == 'success' &&
295
+ needs.verify.result == 'success' &&
296
+ needs.build.result == 'success' &&
297
+ needs.install-smoke.result == 'success' &&
298
+ needs.testpypi-publish.result == 'success' &&
299
+ needs.testpypi-smoke.result == 'success' &&
300
+ needs.publish.result == 'success' &&
301
+ needs.candidate.outputs.mode == 'tag'
302
+ needs: [candidate, verify, build, install-smoke, testpypi-publish, testpypi-smoke, publish]
303
+ runs-on: ubuntu-latest
304
+ environment: github-release
305
+ permissions:
306
+ contents: write
307
+ steps:
308
+ - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
309
+ with:
310
+ persist-credentials: false
311
+ ref: ${{ needs.candidate.outputs.sha }}
312
+ - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
313
+ with:
314
+ artifact-ids: ${{ needs.build.outputs.distributions_artifact_id }}
315
+ path: release/
316
+ merge-multiple: true
317
+ - run: >-
318
+ python tools/verify_release_tag.py
319
+ --tag "${{ needs.candidate.outputs.tag }}"
320
+ --expected-sha "${{ needs.candidate.outputs.sha }}"
321
+ - run: >-
322
+ python tools/verify_release_artifacts.py release/packages
323
+ --name ray-doris
324
+ --version "${{ needs.candidate.outputs.version }}"
325
+ --manifest release/SHA256SUMS
326
+ - name: Publish immutable release assets
327
+ env:
328
+ GH_TOKEN: ${{ github.token }}
329
+ run: >-
330
+ gh release create "${{ needs.candidate.outputs.tag }}"
331
+ release/packages/* release/SHA256SUMS
332
+ --title "ray-doris ${{ needs.candidate.outputs.version }}"
333
+ --notes-file "release-notes/v${{ needs.candidate.outputs.version }}.md"
@@ -0,0 +1,90 @@
1
+ name: Slow distributed integration
2
+
3
+ on:
4
+ workflow_call:
5
+ inputs:
6
+ profile:
7
+ description: Slow integration profile
8
+ required: false
9
+ default: full
10
+ type: string
11
+ row_count:
12
+ description: Rows loaded into each slow integration table
13
+ required: false
14
+ default: "10000"
15
+ type: string
16
+ stress_seconds:
17
+ description: Minimum duration of repeated Flight SQL reads
18
+ required: false
19
+ default: "5"
20
+ type: string
21
+ be_memory_limit:
22
+ description: Memory limit for each Doris BE container
23
+ required: false
24
+ default: 2g
25
+ type: string
26
+ workflow_dispatch:
27
+ inputs:
28
+ profile:
29
+ description: Slow integration profile
30
+ required: true
31
+ default: full
32
+ type: choice
33
+ options: [full, core]
34
+ row_count:
35
+ description: Rows loaded into each slow integration table
36
+ required: true
37
+ default: "10000"
38
+ type: string
39
+ stress_seconds:
40
+ description: Minimum duration of repeated Flight SQL reads
41
+ required: true
42
+ default: "5"
43
+ type: string
44
+ be_memory_limit:
45
+ description: Memory limit for each Doris BE container
46
+ required: true
47
+ default: "2g"
48
+ type: string
49
+ schedule:
50
+ - cron: "17 3 * * 0"
51
+
52
+ permissions:
53
+ contents: read
54
+
55
+ concurrency:
56
+ group: slow-integration-${{ github.ref }}
57
+ cancel-in-progress: false
58
+
59
+ jobs:
60
+ distributed:
61
+ name: Ray and Doris distributed slow integration
62
+ runs-on: [self-hosted, linux, x64, ray-doris-slow-it]
63
+ timeout-minutes: 30
64
+ env:
65
+ RAY_DORIS_SLOW_PROFILE: ${{ inputs.profile || 'full' }}
66
+ RAY_DORIS_ROW_COUNT: ${{ inputs.row_count || '10000' }}
67
+ RAY_DORIS_STRESS_SECONDS: ${{ inputs.stress_seconds || '5' }}
68
+ RAY_DORIS_BE_MEMORY_LIMIT: ${{ inputs.be_memory_limit || '2g' }}
69
+ steps:
70
+ - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
71
+ - name: Run distributed cluster scenario
72
+ env:
73
+ RAY_DORIS_SLOW_LOG_DIR: ${{ runner.temp }}/ray-doris-slow-it
74
+ run: tests/slow_integration/run.sh
75
+ - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
76
+ if: always()
77
+ with:
78
+ name: ray-doris-slow-it-logs
79
+ path: ${{ runner.temp }}/ray-doris-slow-it/*.log
80
+ if-no-files-found: warn
81
+ overwrite: true
82
+ retention-days: 30
83
+ - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
84
+ if: ${{ success() && env.RAY_DORIS_SLOW_PROFILE == 'full' }}
85
+ with:
86
+ name: ray-doris-slow-result-${{ github.sha }}
87
+ path: ${{ runner.temp }}/ray-doris-slow-it/slow-result.json
88
+ if-no-files-found: error
89
+ overwrite: true
90
+ retention-days: 90
@@ -0,0 +1,15 @@
1
+ .DS_Store
2
+ .coverage
3
+ .mypy_cache/
4
+ .pytest_cache/
5
+ .ruff_cache/
6
+ .docs-venv/
7
+ .venv*/
8
+ uv.lock
9
+ __pycache__/
10
+ build/
11
+ doc/_build/
12
+ dist/
13
+ htmlcov/
14
+ *.egg-info/
15
+ *.py[cod]
@@ -0,0 +1,14 @@
1
+ repos:
2
+ - repo: https://github.com/astral-sh/ruff-pre-commit
3
+ rev: v0.12.8
4
+ hooks:
5
+ - id: ruff-check
6
+ args: [--fix]
7
+ - id: ruff-format
8
+ - repo: https://github.com/pre-commit/pre-commit-hooks
9
+ rev: v5.0.0
10
+ hooks:
11
+ - id: check-toml
12
+ - id: check-yaml
13
+ - id: end-of-file-fixer
14
+ - id: trailing-whitespace
@@ -0,0 +1,23 @@
1
+ version: 2
2
+
3
+ build:
4
+ os: ubuntu-24.04
5
+ tools:
6
+ python: "3.12"
7
+ apt_packages:
8
+ - enchant-2
9
+ jobs:
10
+ install:
11
+ - pip install --requirement doc/requirements-doc.lock.txt
12
+ - pip install --no-build-isolation --no-deps --editable .
13
+ build:
14
+ html:
15
+ # This custom command replaces the default RTD Sphinx build. The
16
+ # Makefile enforces warnings-as-errors with -W and --keep-going.
17
+ - >-
18
+ make -C doc html
19
+ SPHINXBUILD=sphinx-build
20
+ HTMLDIR="$READTHEDOCS_OUTPUT/html"
21
+
22
+ sphinx:
23
+ configuration: doc/source/conf.py
@@ -0,0 +1,28 @@
1
+ # Changelog
2
+
3
+ All notable changes to this project are documented in this file.
4
+
5
+ ## Unreleased
6
+
7
+ - Bound Ray support to tested 2.49.2 through 2.56.x DeveloperAPI signatures.
8
+ - Balance tablet splits, avoid draining active MySQL results on early consumer termination, and
9
+ align MySQL and Flight result-schema checks.
10
+ - Reject non-serializable worker options on the driver and redact filters and server details from
11
+ representations, logs, and public errors.
12
+ - Mark Python 3.9 as Alpha legacy compatibility and Flight SQL and automatic transport selection as
13
+ experimental.
14
+ - Validate release ancestry against the `master` default branch.
15
+ - Add environment-referenced credentials that resolve independently on the driver and workers
16
+ without serializing the resolved value.
17
+ - Add a private-CA option and independent timeout for query-plan HTTPS, plus fixed redacted MySQL
18
+ TLS setup errors.
19
+ - Move distributed worker-retry and backend-failure evidence to the minimum-privilege MySQL path.
20
+ - Bind releases to a successful full slow-suite manifest from the exact release commit and workflow
21
+ run while keeping FE cluster HA as a deployment responsibility.
22
+
23
+ ## 0.1.0a1
24
+
25
+ - Add tablet-aware Apache Doris reads through Ray Data's documented V1 Datasource extension API.
26
+ - Add bounded MySQL streaming and optional Arrow Flight SQL transport.
27
+ - Add explicit schema conversion, deterministic planning, and controlled fallback behavior.
28
+ - Add Ray compatibility, real Doris integration, package, and release verification workflows.