ray-doris 1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ray_doris-1.0/.dockerignore +10 -0
- ray_doris-1.0/.github/workflows/ci.yml +163 -0
- ray_doris-1.0/.github/workflows/release.yml +333 -0
- ray_doris-1.0/.github/workflows/slow-integration.yml +90 -0
- ray_doris-1.0/.gitignore +15 -0
- ray_doris-1.0/.pre-commit-config.yaml +14 -0
- ray_doris-1.0/.readthedocs.yaml +23 -0
- ray_doris-1.0/CHANGELOG.md +28 -0
- ray_doris-1.0/CONTRIBUTING.md +139 -0
- ray_doris-1.0/LICENSE +186 -0
- ray_doris-1.0/NOTICE +4 -0
- ray_doris-1.0/PKG-INFO +450 -0
- ray_doris-1.0/README.md +410 -0
- ray_doris-1.0/SECURITY.md +24 -0
- ray_doris-1.0/doc/Makefile +31 -0
- ray_doris-1.0/doc/requirements-doc.lock.txt +158 -0
- ray_doris-1.0/doc/requirements-doc.txt +12 -0
- ray_doris-1.0/doc/source/_static/custom.css +15 -0
- ray_doris-1.0/doc/source/_templates/main-sidebar.html +15 -0
- ray_doris-1.0/doc/source/api/api.md +122 -0
- ray_doris-1.0/doc/source/architecture.md +119 -0
- ray_doris-1.0/doc/source/compatibility.md +94 -0
- ray_doris-1.0/doc/source/conf.py +109 -0
- ray_doris-1.0/doc/source/contributing/index.md +76 -0
- ray_doris-1.0/doc/source/examples/index.md +122 -0
- ray_doris-1.0/doc/source/faq.md +85 -0
- ray_doris-1.0/doc/source/index.md +113 -0
- ray_doris-1.0/doc/source/key-concepts.md +85 -0
- ray_doris-1.0/doc/source/quickstart.md +80 -0
- ray_doris-1.0/doc/source/spelling_wordlist.txt +70 -0
- ray_doris-1.0/doc/source/user-guide/configure-transports.md +109 -0
- ray_doris-1.0/doc/source/user-guide/index.md +76 -0
- ray_doris-1.0/doc/source/user-guide/read-data.md +136 -0
- ray_doris-1.0/doc/source/user-guide/secure-connections.md +151 -0
- ray_doris-1.0/doc/source/user-guide/troubleshooting.md +102 -0
- ray_doris-1.0/doc/source/user-guide/tune-parallelism.md +93 -0
- ray_doris-1.0/doc/source/user-guide/work-with-schemas.md +75 -0
- ray_doris-1.0/doc/source/user-guide/write-data.md +109 -0
- ray_doris-1.0/examples/quickstart.py +21 -0
- ray_doris-1.0/pyproject.toml +82 -0
- ray_doris-1.0/release-notes/v1.0.md +27 -0
- ray_doris-1.0/src/ray_doris/__init__.py +51 -0
- ray_doris-1.0/src/ray_doris/_api.py +73 -0
- ray_doris-1.0/src/ray_doris/_compat.py +117 -0
- ray_doris-1.0/src/ray_doris/_errors.py +49 -0
- ray_doris-1.0/src/ray_doris/_models.py +292 -0
- ray_doris-1.0/src/ray_doris/_planner.py +401 -0
- ray_doris-1.0/src/ray_doris/_readers.py +396 -0
- ray_doris-1.0/src/ray_doris/_schema.py +152 -0
- ray_doris-1.0/src/ray_doris/_sql.py +116 -0
- ray_doris-1.0/src/ray_doris/datasource.py +161 -0
- ray_doris-1.0/src/ray_doris/py.typed +0 -0
- ray_doris-1.0/src/ray_doris/write/__init__.py +15 -0
- ray_doris-1.0/src/ray_doris/write/api.py +51 -0
- ray_doris-1.0/src/ray_doris/write/connection.py +201 -0
- ray_doris-1.0/src/ray_doris/write/datasink.py +242 -0
- ray_doris-1.0/src/ray_doris/write/metadata.py +367 -0
- ray_doris-1.0/src/ray_doris/write/options.py +148 -0
- ray_doris-1.0/src/ray_doris/write/serialization.py +112 -0
- ray_doris-1.0/src/ray_doris/write/stream_load.py +396 -0
- ray_doris-1.0/tests/__init__.py +1 -0
- ray_doris-1.0/tests/contract/__init__.py +1 -0
- ray_doris-1.0/tests/contract/test_write_datasink.py +112 -0
- ray_doris-1.0/tests/integration/conftest.py +355 -0
- ray_doris-1.0/tests/integration/docker/be.Dockerfile +3 -0
- ray_doris-1.0/tests/integration/docker/be.conf +9 -0
- ray_doris-1.0/tests/integration/docker/fe.Dockerfile +3 -0
- ray_doris-1.0/tests/integration/docker/fe.conf +10 -0
- ray_doris-1.0/tests/integration/docker-compose.yml +58 -0
- ray_doris-1.0/tests/integration/test_distributed_read.py +16 -0
- ray_doris-1.0/tests/integration/test_flight_read.py +69 -0
- ray_doris-1.0/tests/integration/test_mysql_read.py +222 -0
- ray_doris-1.0/tests/integration/test_query_plan_errors.py +49 -0
- ray_doris-1.0/tests/integration/test_write.py +193 -0
- ray_doris-1.0/tests/it-ledger.md +44 -0
- ray_doris-1.0/tests/slow_integration/_cluster.py +419 -0
- ray_doris-1.0/tests/slow_integration/conftest.py +46 -0
- ray_doris-1.0/tests/slow_integration/docker/be.Dockerfile +3 -0
- ray_doris-1.0/tests/slow_integration/docker/be.conf +9 -0
- ray_doris-1.0/tests/slow_integration/docker/fe.Dockerfile +3 -0
- ray_doris-1.0/tests/slow_integration/docker/fe.conf +14 -0
- ray_doris-1.0/tests/slow_integration/docker/generate-tls.sh +69 -0
- ray_doris-1.0/tests/slow_integration/docker/haproxy.cfg +78 -0
- ray_doris-1.0/tests/slow_integration/docker/ray.Dockerfile +20 -0
- ray_doris-1.0/tests/slow_integration/docker/write_fault_server.py +61 -0
- ray_doris-1.0/tests/slow_integration/docker-compose.yml +283 -0
- ray_doris-1.0/tests/slow_integration/run.sh +130 -0
- ray_doris-1.0/tests/slow_integration/test_distributed_cluster.py +652 -0
- ray_doris-1.0/tests/slow_integration/write_result.py +50 -0
- ray_doris-1.0/tests/unit/test_api.py +59 -0
- ray_doris-1.0/tests/unit/test_compat.py +106 -0
- ray_doris-1.0/tests/unit/test_datasource.py +126 -0
- ray_doris-1.0/tests/unit/test_models.py +193 -0
- ray_doris-1.0/tests/unit/test_planner.py +523 -0
- ray_doris-1.0/tests/unit/test_readers.py +673 -0
- ray_doris-1.0/tests/unit/test_release.py +200 -0
- ray_doris-1.0/tests/unit/test_schema.py +82 -0
- ray_doris-1.0/tests/unit/test_sql.py +64 -0
- ray_doris-1.0/tests/unit/test_write.py +536 -0
- ray_doris-1.0/tools/check_docs.py +196 -0
- ray_doris-1.0/tools/check_release.py +179 -0
- ray_doris-1.0/tools/check_release_workflow.py +154 -0
- ray_doris-1.0/tools/check_slow_result.py +101 -0
- ray_doris-1.0/tools/check_workflow_pins.py +48 -0
- ray_doris-1.0/tools/verify_release_artifacts.py +91 -0
- ray_doris-1.0/tools/verify_release_tag.py +43 -0
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
workflow_call:
|
|
5
|
+
inputs:
|
|
6
|
+
candidate_sha:
|
|
7
|
+
description: Immutable release candidate commit to test
|
|
8
|
+
required: false
|
|
9
|
+
type: string
|
|
10
|
+
pull_request:
|
|
11
|
+
push:
|
|
12
|
+
branches: [master]
|
|
13
|
+
schedule:
|
|
14
|
+
- cron: "17 3 * * 1"
|
|
15
|
+
|
|
16
|
+
concurrency:
|
|
17
|
+
group: ci-${{ github.workflow }}-${{ github.ref }}
|
|
18
|
+
cancel-in-progress: true
|
|
19
|
+
|
|
20
|
+
jobs:
|
|
21
|
+
unit:
|
|
22
|
+
name: Python ${{ matrix.python }} / Ray ${{ matrix.ray }}
|
|
23
|
+
runs-on: ubuntu-latest
|
|
24
|
+
timeout-minutes: 15
|
|
25
|
+
strategy:
|
|
26
|
+
fail-fast: false
|
|
27
|
+
matrix:
|
|
28
|
+
include:
|
|
29
|
+
- python: "3.9"
|
|
30
|
+
ray: "2.49.2"
|
|
31
|
+
- python: "3.10"
|
|
32
|
+
ray: "2.55.1"
|
|
33
|
+
- python: "3.12"
|
|
34
|
+
ray: "2.56.1"
|
|
35
|
+
steps:
|
|
36
|
+
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
|
|
37
|
+
with:
|
|
38
|
+
persist-credentials: false
|
|
39
|
+
ref: ${{ inputs.candidate_sha || github.sha }}
|
|
40
|
+
- uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0
|
|
41
|
+
with:
|
|
42
|
+
python-version: ${{ matrix.python }}
|
|
43
|
+
- run: uv venv --python ${{ matrix.python }}
|
|
44
|
+
- run: uv pip install -e ".[test]" "ray[data]==${{ matrix.ray }}"
|
|
45
|
+
- run: .venv/bin/python -m pytest tests/unit tests/contract --cov=ray_doris --cov-report=term-missing
|
|
46
|
+
|
|
47
|
+
quality:
|
|
48
|
+
name: Static quality
|
|
49
|
+
runs-on: ubuntu-latest
|
|
50
|
+
timeout-minutes: 10
|
|
51
|
+
steps:
|
|
52
|
+
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
|
|
53
|
+
with:
|
|
54
|
+
persist-credentials: false
|
|
55
|
+
ref: ${{ inputs.candidate_sha || github.sha }}
|
|
56
|
+
- uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0
|
|
57
|
+
with:
|
|
58
|
+
python-version: "3.12"
|
|
59
|
+
- run: uv venv --python 3.12
|
|
60
|
+
- run: uv pip install -e ".[dev,flight]" "ray[data]==2.56.1"
|
|
61
|
+
- run: .venv/bin/ruff format --check .
|
|
62
|
+
- run: .venv/bin/ruff check .
|
|
63
|
+
- run: .venv/bin/mypy
|
|
64
|
+
- run: .venv/bin/pre-commit run --all-files
|
|
65
|
+
- run: >-
|
|
66
|
+
.venv/bin/python tools/check_workflow_pins.py
|
|
67
|
+
.github/workflows/ci.yml
|
|
68
|
+
.github/workflows/release.yml
|
|
69
|
+
.github/workflows/slow-integration.yml
|
|
70
|
+
- run: >-
|
|
71
|
+
.venv/bin/python tools/check_release_workflow.py
|
|
72
|
+
.github/workflows/ci.yml .github/workflows/release.yml
|
|
73
|
+
|
|
74
|
+
docs:
|
|
75
|
+
name: Documentation
|
|
76
|
+
runs-on: ubuntu-latest
|
|
77
|
+
timeout-minutes: 15
|
|
78
|
+
steps:
|
|
79
|
+
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
|
|
80
|
+
with:
|
|
81
|
+
persist-credentials: false
|
|
82
|
+
ref: ${{ inputs.candidate_sha || github.sha }}
|
|
83
|
+
- uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0
|
|
84
|
+
with:
|
|
85
|
+
python-version: "3.12"
|
|
86
|
+
- name: Install spelling system dependency
|
|
87
|
+
run: |
|
|
88
|
+
sudo apt-get update
|
|
89
|
+
sudo apt-get install --yes enchant-2
|
|
90
|
+
- name: Install documentation environment
|
|
91
|
+
run: |
|
|
92
|
+
uv venv .docs-venv --python 3.12
|
|
93
|
+
uv pip install \
|
|
94
|
+
--python .docs-venv/bin/python \
|
|
95
|
+
--requirement doc/requirements-doc.lock.txt
|
|
96
|
+
uv pip install \
|
|
97
|
+
--python .docs-venv/bin/python \
|
|
98
|
+
--no-build-isolation \
|
|
99
|
+
--no-deps \
|
|
100
|
+
--editable .
|
|
101
|
+
- name: Check documentation contracts
|
|
102
|
+
run: .docs-venv/bin/python tools/check_docs.py
|
|
103
|
+
- name: Build strict documentation
|
|
104
|
+
run: make -C doc strict SPHINXBUILD=../.docs-venv/bin/sphinx-build
|
|
105
|
+
- name: Check documentation spelling
|
|
106
|
+
run: make -C doc spelling SPHINXBUILD=../.docs-venv/bin/sphinx-build
|
|
107
|
+
- uses: lycheeverse/lychee-action@e7477775783ea5526144ba13e8db5eec57747ce8 # v2.4.1
|
|
108
|
+
with:
|
|
109
|
+
args: >-
|
|
110
|
+
--no-progress
|
|
111
|
+
--max-retries 3
|
|
112
|
+
README.md CONTRIBUTING.md SECURITY.md CHANGELOG.md doc/source
|
|
113
|
+
|
|
114
|
+
integration:
|
|
115
|
+
name: Doris 4.0.6 integration
|
|
116
|
+
runs-on: ubuntu-latest
|
|
117
|
+
timeout-minutes: 25
|
|
118
|
+
steps:
|
|
119
|
+
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
|
|
120
|
+
with:
|
|
121
|
+
persist-credentials: false
|
|
122
|
+
ref: ${{ inputs.candidate_sha || github.sha }}
|
|
123
|
+
- uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0
|
|
124
|
+
with:
|
|
125
|
+
python-version: "3.12"
|
|
126
|
+
- run: uv venv --python 3.12
|
|
127
|
+
- run: uv pip install -e ".[test,flight]" "ray[data]==2.56.1"
|
|
128
|
+
- run: docker compose -f tests/integration/docker-compose.yml up -d --build
|
|
129
|
+
- run: .venv/bin/python -m pytest tests/integration -v
|
|
130
|
+
- name: Capture Doris logs
|
|
131
|
+
if: always()
|
|
132
|
+
run: docker compose -f tests/integration/docker-compose.yml logs --no-color > doris-it.log
|
|
133
|
+
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
|
134
|
+
if: always()
|
|
135
|
+
with:
|
|
136
|
+
name: doris-it-log
|
|
137
|
+
path: doris-it.log
|
|
138
|
+
- name: Stop Doris
|
|
139
|
+
if: always()
|
|
140
|
+
run: docker compose -f tests/integration/docker-compose.yml down -v --rmi local
|
|
141
|
+
|
|
142
|
+
package:
|
|
143
|
+
name: Package
|
|
144
|
+
runs-on: ubuntu-latest
|
|
145
|
+
timeout-minutes: 15
|
|
146
|
+
steps:
|
|
147
|
+
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
|
|
148
|
+
with:
|
|
149
|
+
persist-credentials: false
|
|
150
|
+
ref: ${{ inputs.candidate_sha || github.sha }}
|
|
151
|
+
- uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0
|
|
152
|
+
with:
|
|
153
|
+
python-version: "3.12"
|
|
154
|
+
- run: uv venv --python 3.12
|
|
155
|
+
- run: uv pip install build twine
|
|
156
|
+
- run: .venv/bin/python -m build
|
|
157
|
+
- run: .venv/bin/twine check dist/*
|
|
158
|
+
- run: uv venv --python 3.12 .venv-package
|
|
159
|
+
- run: uv pip install --python .venv-package/bin/python dist/*.whl
|
|
160
|
+
- run: .venv-package/bin/python -c "import ray_doris; print(ray_doris.__version__)"
|
|
161
|
+
- run: uv venv --python 3.12 .venv-sdist
|
|
162
|
+
- run: uv pip install --python .venv-sdist/bin/python dist/*.tar.gz
|
|
163
|
+
- run: .venv-sdist/bin/python -c "import ray_doris; print(ray_doris.__version__)"
|
|
@@ -0,0 +1,333 @@
|
|
|
1
|
+
name: Release
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags: ["v*"]
|
|
6
|
+
workflow_dispatch:
|
|
7
|
+
inputs:
|
|
8
|
+
candidate_sha:
|
|
9
|
+
description: Full master-reachable commit SHA to validate without publishing
|
|
10
|
+
required: true
|
|
11
|
+
type: string
|
|
12
|
+
expected_version:
|
|
13
|
+
description: Exact package version expected at the candidate commit
|
|
14
|
+
required: true
|
|
15
|
+
type: string
|
|
16
|
+
|
|
17
|
+
permissions:
|
|
18
|
+
contents: read
|
|
19
|
+
actions: read
|
|
20
|
+
|
|
21
|
+
concurrency:
|
|
22
|
+
group: release-${{ github.event_name == 'push' && github.ref || inputs.candidate_sha }}
|
|
23
|
+
cancel-in-progress: false
|
|
24
|
+
|
|
25
|
+
jobs:
|
|
26
|
+
candidate:
|
|
27
|
+
name: Validate release candidate
|
|
28
|
+
runs-on: ubuntu-latest
|
|
29
|
+
outputs:
|
|
30
|
+
mode: ${{ steps.candidate.outputs.mode }}
|
|
31
|
+
sha: ${{ steps.candidate.outputs.sha }}
|
|
32
|
+
version: ${{ steps.candidate.outputs.version }}
|
|
33
|
+
tag: ${{ steps.candidate.outputs.tag }}
|
|
34
|
+
master_sha: ${{ steps.candidate.outputs.master_sha }}
|
|
35
|
+
steps:
|
|
36
|
+
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
|
|
37
|
+
with:
|
|
38
|
+
fetch-depth: 0
|
|
39
|
+
persist-credentials: false
|
|
40
|
+
ref: master
|
|
41
|
+
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
|
42
|
+
with:
|
|
43
|
+
python-version: "3.12"
|
|
44
|
+
- name: Fetch protected master and tag identities
|
|
45
|
+
run: >-
|
|
46
|
+
git fetch --force origin
|
|
47
|
+
+refs/heads/master:refs/remotes/origin/master
|
|
48
|
+
+refs/tags/*:refs/tags/*
|
|
49
|
+
- name: Resolve immutable candidate
|
|
50
|
+
id: candidate
|
|
51
|
+
env:
|
|
52
|
+
CANDIDATE_REF: ${{ github.event_name == 'workflow_dispatch' && inputs.candidate_sha || github.ref }}
|
|
53
|
+
EXPECTED_VERSION: ${{ github.event_name == 'workflow_dispatch' && inputs.expected_version || '' }}
|
|
54
|
+
MODE: ${{ github.event_name == 'workflow_dispatch' && 'dry-run' || 'tag' }}
|
|
55
|
+
RELEASE_TAG: ${{ github.event_name == 'push' && github.ref_name || '' }}
|
|
56
|
+
EVENT_SHA: ${{ github.event_name == 'push' && github.event.after || '' }}
|
|
57
|
+
EVENT_CREATED: ${{ github.event_name == 'push' && github.event.created || false }}
|
|
58
|
+
EVENT_DELETED: ${{ github.event_name == 'push' && github.event.deleted || false }}
|
|
59
|
+
EVENT_FORCED: ${{ github.event_name == 'push' && github.event.forced || false }}
|
|
60
|
+
run: >-
|
|
61
|
+
python tools/check_release.py
|
|
62
|
+
--mode "${MODE}"
|
|
63
|
+
--candidate-ref "${CANDIDATE_REF}"
|
|
64
|
+
--expected-version "${EXPECTED_VERSION}"
|
|
65
|
+
--tag "${RELEASE_TAG}"
|
|
66
|
+
--master-ref origin/master
|
|
67
|
+
--event-sha "${EVENT_SHA}"
|
|
68
|
+
--event-created "${EVENT_CREATED}"
|
|
69
|
+
--event-deleted "${EVENT_DELETED}"
|
|
70
|
+
--event-forced "${EVENT_FORCED}"
|
|
71
|
+
--github-output "${GITHUB_OUTPUT}"
|
|
72
|
+
|
|
73
|
+
verify:
|
|
74
|
+
name: Candidate CI gates
|
|
75
|
+
needs: candidate
|
|
76
|
+
uses: ./.github/workflows/ci.yml
|
|
77
|
+
with:
|
|
78
|
+
candidate_sha: ${{ needs.candidate.outputs.sha }}
|
|
79
|
+
|
|
80
|
+
build:
|
|
81
|
+
name: Build and verify distributions
|
|
82
|
+
needs: [candidate, verify]
|
|
83
|
+
runs-on: ubuntu-latest
|
|
84
|
+
outputs:
|
|
85
|
+
distributions_artifact_id: ${{ steps.upload.outputs.artifact-id }}
|
|
86
|
+
steps:
|
|
87
|
+
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
|
|
88
|
+
with:
|
|
89
|
+
persist-credentials: false
|
|
90
|
+
ref: ${{ needs.candidate.outputs.sha }}
|
|
91
|
+
- uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0
|
|
92
|
+
with:
|
|
93
|
+
enable-cache: true
|
|
94
|
+
python-version: "3.12"
|
|
95
|
+
- run: uv venv --python 3.12
|
|
96
|
+
- run: uv pip install --python .venv/bin/python "build==1.3.0" "twine==6.2.0"
|
|
97
|
+
- run: .venv/bin/python -m build --outdir release-artifacts/packages
|
|
98
|
+
- run: .venv/bin/twine check release-artifacts/packages/*
|
|
99
|
+
- run: >-
|
|
100
|
+
.venv/bin/python tools/verify_release_artifacts.py release-artifacts/packages
|
|
101
|
+
--name ray-doris
|
|
102
|
+
--version "${{ needs.candidate.outputs.version }}"
|
|
103
|
+
--manifest release-artifacts/SHA256SUMS
|
|
104
|
+
--prepare
|
|
105
|
+
- name: Upload immutable distributions
|
|
106
|
+
id: upload
|
|
107
|
+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
|
108
|
+
with:
|
|
109
|
+
name: distributions
|
|
110
|
+
path: release-artifacts/
|
|
111
|
+
if-no-files-found: error
|
|
112
|
+
|
|
113
|
+
install-smoke:
|
|
114
|
+
name: Install ${{ matrix.distribution }} ${{ matrix.profile }}
|
|
115
|
+
needs: [candidate, verify, build]
|
|
116
|
+
runs-on: ubuntu-latest
|
|
117
|
+
strategy:
|
|
118
|
+
fail-fast: false
|
|
119
|
+
matrix:
|
|
120
|
+
include:
|
|
121
|
+
- profile: base
|
|
122
|
+
extra: ""
|
|
123
|
+
distribution: wheel
|
|
124
|
+
- profile: flight
|
|
125
|
+
extra: "[flight]"
|
|
126
|
+
distribution: wheel
|
|
127
|
+
- profile: base
|
|
128
|
+
extra: ""
|
|
129
|
+
distribution: sdist
|
|
130
|
+
- profile: flight
|
|
131
|
+
extra: "[flight]"
|
|
132
|
+
distribution: sdist
|
|
133
|
+
steps:
|
|
134
|
+
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
|
|
135
|
+
with:
|
|
136
|
+
persist-credentials: false
|
|
137
|
+
ref: ${{ needs.candidate.outputs.sha }}
|
|
138
|
+
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
|
139
|
+
with:
|
|
140
|
+
python-version: "3.12"
|
|
141
|
+
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
|
|
142
|
+
with:
|
|
143
|
+
artifact-ids: ${{ needs.build.outputs.distributions_artifact_id }}
|
|
144
|
+
path: release/
|
|
145
|
+
merge-multiple: true
|
|
146
|
+
- run: >-
|
|
147
|
+
python tools/verify_release_artifacts.py release/packages
|
|
148
|
+
--name ray-doris
|
|
149
|
+
--version "${{ needs.candidate.outputs.version }}"
|
|
150
|
+
--manifest release/SHA256SUMS
|
|
151
|
+
- name: Install built distribution
|
|
152
|
+
env:
|
|
153
|
+
EXTRA: ${{ matrix.extra }}
|
|
154
|
+
DISTRIBUTION: ${{ matrix.distribution }}
|
|
155
|
+
run: |
|
|
156
|
+
if [ "$DISTRIBUTION" = "wheel" ]; then
|
|
157
|
+
package="$(find release/packages -maxdepth 1 -type f -name '*.whl')"
|
|
158
|
+
else
|
|
159
|
+
package="$(find release/packages -maxdepth 1 -type f -name '*.tar.gz')"
|
|
160
|
+
fi
|
|
161
|
+
test -n "$package"
|
|
162
|
+
python -m pip install "ray-doris${EXTRA} @ file://${GITHUB_WORKSPACE}/${package}"
|
|
163
|
+
- run: python -c "import ray_doris; print(ray_doris.__version__)"
|
|
164
|
+
|
|
165
|
+
testpypi-publish:
|
|
166
|
+
name: Publish formal artifact to TestPyPI
|
|
167
|
+
if: >-
|
|
168
|
+
always() &&
|
|
169
|
+
github.event_name == 'push' &&
|
|
170
|
+
startsWith(github.ref, 'refs/tags/v') &&
|
|
171
|
+
needs.candidate.result == 'success' &&
|
|
172
|
+
needs.verify.result == 'success' &&
|
|
173
|
+
needs.build.result == 'success' &&
|
|
174
|
+
needs.install-smoke.result == 'success' &&
|
|
175
|
+
needs.candidate.outputs.mode == 'tag'
|
|
176
|
+
needs: [candidate, verify, build, install-smoke]
|
|
177
|
+
runs-on: ubuntu-latest
|
|
178
|
+
environment: testpypi
|
|
179
|
+
permissions:
|
|
180
|
+
contents: read
|
|
181
|
+
id-token: write
|
|
182
|
+
steps:
|
|
183
|
+
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
|
|
184
|
+
with:
|
|
185
|
+
persist-credentials: false
|
|
186
|
+
ref: ${{ needs.candidate.outputs.sha }}
|
|
187
|
+
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
|
188
|
+
with:
|
|
189
|
+
python-version: "3.12"
|
|
190
|
+
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
|
|
191
|
+
with:
|
|
192
|
+
artifact-ids: ${{ needs.build.outputs.distributions_artifact_id }}
|
|
193
|
+
path: release/
|
|
194
|
+
merge-multiple: true
|
|
195
|
+
- run: >-
|
|
196
|
+
python tools/verify_release_tag.py
|
|
197
|
+
--tag "${{ needs.candidate.outputs.tag }}"
|
|
198
|
+
--expected-sha "${{ needs.candidate.outputs.sha }}"
|
|
199
|
+
- run: >-
|
|
200
|
+
python tools/verify_release_artifacts.py release/packages
|
|
201
|
+
--name ray-doris
|
|
202
|
+
--version "${{ needs.candidate.outputs.version }}"
|
|
203
|
+
--manifest release/SHA256SUMS
|
|
204
|
+
- uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2
|
|
205
|
+
with:
|
|
206
|
+
packages-dir: release/packages/
|
|
207
|
+
repository-url: https://test.pypi.org/legacy/
|
|
208
|
+
|
|
209
|
+
testpypi-smoke:
|
|
210
|
+
name: TestPyPI smoke (${{ matrix.profile }})
|
|
211
|
+
if: >-
|
|
212
|
+
always() &&
|
|
213
|
+
github.event_name == 'push' &&
|
|
214
|
+
startsWith(github.ref, 'refs/tags/v') &&
|
|
215
|
+
needs.candidate.result == 'success' &&
|
|
216
|
+
needs.testpypi-publish.result == 'success' &&
|
|
217
|
+
needs.candidate.outputs.mode == 'tag'
|
|
218
|
+
needs: [candidate, testpypi-publish]
|
|
219
|
+
runs-on: ubuntu-latest
|
|
220
|
+
strategy:
|
|
221
|
+
fail-fast: false
|
|
222
|
+
matrix:
|
|
223
|
+
include:
|
|
224
|
+
- profile: base
|
|
225
|
+
extra: ""
|
|
226
|
+
- profile: flight
|
|
227
|
+
extra: "[flight]"
|
|
228
|
+
steps:
|
|
229
|
+
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
|
230
|
+
with:
|
|
231
|
+
python-version: "3.12"
|
|
232
|
+
- name: Install from TestPyPI with PyPI dependency fallback
|
|
233
|
+
env:
|
|
234
|
+
EXTRA: ${{ matrix.extra }}
|
|
235
|
+
VERSION: ${{ needs.candidate.outputs.version }}
|
|
236
|
+
run: >-
|
|
237
|
+
python -m pip install
|
|
238
|
+
--index-url https://test.pypi.org/simple/
|
|
239
|
+
--extra-index-url https://pypi.org/simple/
|
|
240
|
+
"ray-doris${EXTRA}==${VERSION}"
|
|
241
|
+
- run: python -c "import ray_doris; print(ray_doris.__version__)"
|
|
242
|
+
|
|
243
|
+
publish:
|
|
244
|
+
name: Publish to PyPI
|
|
245
|
+
if: >-
|
|
246
|
+
always() &&
|
|
247
|
+
github.event_name == 'push' &&
|
|
248
|
+
startsWith(github.ref, 'refs/tags/v') &&
|
|
249
|
+
needs.candidate.result == 'success' &&
|
|
250
|
+
needs.verify.result == 'success' &&
|
|
251
|
+
needs.build.result == 'success' &&
|
|
252
|
+
needs.install-smoke.result == 'success' &&
|
|
253
|
+
needs.testpypi-publish.result == 'success' &&
|
|
254
|
+
needs.testpypi-smoke.result == 'success' &&
|
|
255
|
+
needs.candidate.outputs.mode == 'tag'
|
|
256
|
+
needs: [candidate, verify, build, install-smoke, testpypi-publish, testpypi-smoke]
|
|
257
|
+
runs-on: ubuntu-latest
|
|
258
|
+
environment: pypi
|
|
259
|
+
permissions:
|
|
260
|
+
contents: read
|
|
261
|
+
id-token: write
|
|
262
|
+
steps:
|
|
263
|
+
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
|
|
264
|
+
with:
|
|
265
|
+
persist-credentials: false
|
|
266
|
+
ref: ${{ needs.candidate.outputs.sha }}
|
|
267
|
+
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
|
268
|
+
with:
|
|
269
|
+
python-version: "3.12"
|
|
270
|
+
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
|
|
271
|
+
with:
|
|
272
|
+
artifact-ids: ${{ needs.build.outputs.distributions_artifact_id }}
|
|
273
|
+
path: release/
|
|
274
|
+
merge-multiple: true
|
|
275
|
+
- run: >-
|
|
276
|
+
python tools/verify_release_tag.py
|
|
277
|
+
--tag "${{ needs.candidate.outputs.tag }}"
|
|
278
|
+
--expected-sha "${{ needs.candidate.outputs.sha }}"
|
|
279
|
+
- run: >-
|
|
280
|
+
python tools/verify_release_artifacts.py release/packages
|
|
281
|
+
--name ray-doris
|
|
282
|
+
--version "${{ needs.candidate.outputs.version }}"
|
|
283
|
+
--manifest release/SHA256SUMS
|
|
284
|
+
- uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2
|
|
285
|
+
with:
|
|
286
|
+
packages-dir: release/packages/
|
|
287
|
+
|
|
288
|
+
github-release:
|
|
289
|
+
name: Create GitHub Release
|
|
290
|
+
if: >-
|
|
291
|
+
always() &&
|
|
292
|
+
github.event_name == 'push' &&
|
|
293
|
+
startsWith(github.ref, 'refs/tags/v') &&
|
|
294
|
+
needs.candidate.result == 'success' &&
|
|
295
|
+
needs.verify.result == 'success' &&
|
|
296
|
+
needs.build.result == 'success' &&
|
|
297
|
+
needs.install-smoke.result == 'success' &&
|
|
298
|
+
needs.testpypi-publish.result == 'success' &&
|
|
299
|
+
needs.testpypi-smoke.result == 'success' &&
|
|
300
|
+
needs.publish.result == 'success' &&
|
|
301
|
+
needs.candidate.outputs.mode == 'tag'
|
|
302
|
+
needs: [candidate, verify, build, install-smoke, testpypi-publish, testpypi-smoke, publish]
|
|
303
|
+
runs-on: ubuntu-latest
|
|
304
|
+
environment: github-release
|
|
305
|
+
permissions:
|
|
306
|
+
contents: write
|
|
307
|
+
steps:
|
|
308
|
+
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
|
|
309
|
+
with:
|
|
310
|
+
persist-credentials: false
|
|
311
|
+
ref: ${{ needs.candidate.outputs.sha }}
|
|
312
|
+
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
|
|
313
|
+
with:
|
|
314
|
+
artifact-ids: ${{ needs.build.outputs.distributions_artifact_id }}
|
|
315
|
+
path: release/
|
|
316
|
+
merge-multiple: true
|
|
317
|
+
- run: >-
|
|
318
|
+
python tools/verify_release_tag.py
|
|
319
|
+
--tag "${{ needs.candidate.outputs.tag }}"
|
|
320
|
+
--expected-sha "${{ needs.candidate.outputs.sha }}"
|
|
321
|
+
- run: >-
|
|
322
|
+
python tools/verify_release_artifacts.py release/packages
|
|
323
|
+
--name ray-doris
|
|
324
|
+
--version "${{ needs.candidate.outputs.version }}"
|
|
325
|
+
--manifest release/SHA256SUMS
|
|
326
|
+
- name: Publish immutable release assets
|
|
327
|
+
env:
|
|
328
|
+
GH_TOKEN: ${{ github.token }}
|
|
329
|
+
run: >-
|
|
330
|
+
gh release create "${{ needs.candidate.outputs.tag }}"
|
|
331
|
+
release/packages/* release/SHA256SUMS
|
|
332
|
+
--title "ray-doris ${{ needs.candidate.outputs.version }}"
|
|
333
|
+
--notes-file "release-notes/v${{ needs.candidate.outputs.version }}.md"
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
name: Slow distributed integration
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
workflow_call:
|
|
5
|
+
inputs:
|
|
6
|
+
profile:
|
|
7
|
+
description: Slow integration profile
|
|
8
|
+
required: false
|
|
9
|
+
default: full
|
|
10
|
+
type: string
|
|
11
|
+
row_count:
|
|
12
|
+
description: Rows loaded into each slow integration table
|
|
13
|
+
required: false
|
|
14
|
+
default: "10000"
|
|
15
|
+
type: string
|
|
16
|
+
stress_seconds:
|
|
17
|
+
description: Minimum duration of repeated Flight SQL reads
|
|
18
|
+
required: false
|
|
19
|
+
default: "5"
|
|
20
|
+
type: string
|
|
21
|
+
be_memory_limit:
|
|
22
|
+
description: Memory limit for each Doris BE container
|
|
23
|
+
required: false
|
|
24
|
+
default: 2g
|
|
25
|
+
type: string
|
|
26
|
+
workflow_dispatch:
|
|
27
|
+
inputs:
|
|
28
|
+
profile:
|
|
29
|
+
description: Slow integration profile
|
|
30
|
+
required: true
|
|
31
|
+
default: full
|
|
32
|
+
type: choice
|
|
33
|
+
options: [full, core]
|
|
34
|
+
row_count:
|
|
35
|
+
description: Rows loaded into each slow integration table
|
|
36
|
+
required: true
|
|
37
|
+
default: "10000"
|
|
38
|
+
type: string
|
|
39
|
+
stress_seconds:
|
|
40
|
+
description: Minimum duration of repeated Flight SQL reads
|
|
41
|
+
required: true
|
|
42
|
+
default: "5"
|
|
43
|
+
type: string
|
|
44
|
+
be_memory_limit:
|
|
45
|
+
description: Memory limit for each Doris BE container
|
|
46
|
+
required: true
|
|
47
|
+
default: "2g"
|
|
48
|
+
type: string
|
|
49
|
+
schedule:
|
|
50
|
+
- cron: "17 3 * * 0"
|
|
51
|
+
|
|
52
|
+
permissions:
|
|
53
|
+
contents: read
|
|
54
|
+
|
|
55
|
+
concurrency:
|
|
56
|
+
group: slow-integration-${{ github.ref }}
|
|
57
|
+
cancel-in-progress: false
|
|
58
|
+
|
|
59
|
+
jobs:
|
|
60
|
+
distributed:
|
|
61
|
+
name: Ray and Doris distributed slow integration
|
|
62
|
+
runs-on: [self-hosted, linux, x64, ray-doris-slow-it]
|
|
63
|
+
timeout-minutes: 30
|
|
64
|
+
env:
|
|
65
|
+
RAY_DORIS_SLOW_PROFILE: ${{ inputs.profile || 'full' }}
|
|
66
|
+
RAY_DORIS_ROW_COUNT: ${{ inputs.row_count || '10000' }}
|
|
67
|
+
RAY_DORIS_STRESS_SECONDS: ${{ inputs.stress_seconds || '5' }}
|
|
68
|
+
RAY_DORIS_BE_MEMORY_LIMIT: ${{ inputs.be_memory_limit || '2g' }}
|
|
69
|
+
steps:
|
|
70
|
+
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
|
|
71
|
+
- name: Run distributed cluster scenario
|
|
72
|
+
env:
|
|
73
|
+
RAY_DORIS_SLOW_LOG_DIR: ${{ runner.temp }}/ray-doris-slow-it
|
|
74
|
+
run: tests/slow_integration/run.sh
|
|
75
|
+
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
|
76
|
+
if: always()
|
|
77
|
+
with:
|
|
78
|
+
name: ray-doris-slow-it-logs
|
|
79
|
+
path: ${{ runner.temp }}/ray-doris-slow-it/*.log
|
|
80
|
+
if-no-files-found: warn
|
|
81
|
+
overwrite: true
|
|
82
|
+
retention-days: 30
|
|
83
|
+
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
|
84
|
+
if: ${{ success() && env.RAY_DORIS_SLOW_PROFILE == 'full' }}
|
|
85
|
+
with:
|
|
86
|
+
name: ray-doris-slow-result-${{ github.sha }}
|
|
87
|
+
path: ${{ runner.temp }}/ray-doris-slow-it/slow-result.json
|
|
88
|
+
if-no-files-found: error
|
|
89
|
+
overwrite: true
|
|
90
|
+
retention-days: 90
|
ray_doris-1.0/.gitignore
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
repos:
|
|
2
|
+
- repo: https://github.com/astral-sh/ruff-pre-commit
|
|
3
|
+
rev: v0.12.8
|
|
4
|
+
hooks:
|
|
5
|
+
- id: ruff-check
|
|
6
|
+
args: [--fix]
|
|
7
|
+
- id: ruff-format
|
|
8
|
+
- repo: https://github.com/pre-commit/pre-commit-hooks
|
|
9
|
+
rev: v5.0.0
|
|
10
|
+
hooks:
|
|
11
|
+
- id: check-toml
|
|
12
|
+
- id: check-yaml
|
|
13
|
+
- id: end-of-file-fixer
|
|
14
|
+
- id: trailing-whitespace
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
version: 2
|
|
2
|
+
|
|
3
|
+
build:
|
|
4
|
+
os: ubuntu-24.04
|
|
5
|
+
tools:
|
|
6
|
+
python: "3.12"
|
|
7
|
+
apt_packages:
|
|
8
|
+
- enchant-2
|
|
9
|
+
jobs:
|
|
10
|
+
install:
|
|
11
|
+
- pip install --requirement doc/requirements-doc.lock.txt
|
|
12
|
+
- pip install --no-build-isolation --no-deps --editable .
|
|
13
|
+
build:
|
|
14
|
+
html:
|
|
15
|
+
# This custom command replaces the default RTD Sphinx build. The
|
|
16
|
+
# Makefile enforces warnings-as-errors with -W and --keep-going.
|
|
17
|
+
- >-
|
|
18
|
+
make -C doc html
|
|
19
|
+
SPHINXBUILD=sphinx-build
|
|
20
|
+
HTMLDIR="$READTHEDOCS_OUTPUT/html"
|
|
21
|
+
|
|
22
|
+
sphinx:
|
|
23
|
+
configuration: doc/source/conf.py
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project are documented in this file.
|
|
4
|
+
|
|
5
|
+
## Unreleased
|
|
6
|
+
|
|
7
|
+
- Bound Ray support to tested 2.49.2 through 2.56.x DeveloperAPI signatures.
|
|
8
|
+
- Balance tablet splits, avoid draining active MySQL results on early consumer termination, and
|
|
9
|
+
align MySQL and Flight result-schema checks.
|
|
10
|
+
- Reject non-serializable worker options on the driver and redact filters and server details from
|
|
11
|
+
representations, logs, and public errors.
|
|
12
|
+
- Mark Python 3.9 as Alpha legacy compatibility and Flight SQL and automatic transport selection as
|
|
13
|
+
experimental.
|
|
14
|
+
- Validate release ancestry against the `master` default branch.
|
|
15
|
+
- Add environment-referenced credentials that resolve independently on the driver and workers
|
|
16
|
+
without serializing the resolved value.
|
|
17
|
+
- Add a private-CA option and independent timeout for query-plan HTTPS, plus fixed redacted MySQL
|
|
18
|
+
TLS setup errors.
|
|
19
|
+
- Move distributed worker-retry and backend-failure evidence to the minimum-privilege MySQL path.
|
|
20
|
+
- Bind releases to a successful full slow-suite manifest from the exact release commit and workflow
|
|
21
|
+
run while keeping FE cluster HA as a deployment responsibility.
|
|
22
|
+
|
|
23
|
+
## 0.1.0a1
|
|
24
|
+
|
|
25
|
+
- Add tablet-aware Apache Doris reads through Ray Data's documented V1 Datasource extension API.
|
|
26
|
+
- Add bounded MySQL streaming and optional Arrow Flight SQL transport.
|
|
27
|
+
- Add explicit schema conversion, deterministic planning, and controlled fallback behavior.
|
|
28
|
+
- Add Ray compatibility, real Doris integration, package, and release verification workflows.
|