ray-elasticsearch 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ray_elasticsearch-0.1.0/.github/dependabot.yml +10 -0
- ray_elasticsearch-0.1.0/.github/release.yml +16 -0
- ray_elasticsearch-0.1.0/.github/workflows/ci.yml +185 -0
- ray_elasticsearch-0.1.0/.gitignore +167 -0
- ray_elasticsearch-0.1.0/LICENSE +21 -0
- ray_elasticsearch-0.1.0/PKG-INFO +200 -0
- ray_elasticsearch-0.1.0/README.md +169 -0
- ray_elasticsearch-0.1.0/pyproject.toml +78 -0
- ray_elasticsearch-0.1.0/ray_elasticsearch/__init__.py +330 -0
- ray_elasticsearch-0.1.0/ray_elasticsearch/test_dummy.py +3 -0
- ray_elasticsearch-0.1.0/ray_elasticsearch.egg-info/PKG-INFO +200 -0
- ray_elasticsearch-0.1.0/ray_elasticsearch.egg-info/SOURCES.txt +14 -0
- ray_elasticsearch-0.1.0/ray_elasticsearch.egg-info/dependency_links.txt +1 -0
- ray_elasticsearch-0.1.0/ray_elasticsearch.egg-info/requires.txt +12 -0
- ray_elasticsearch-0.1.0/ray_elasticsearch.egg-info/top_level.txt +2 -0
- ray_elasticsearch-0.1.0/setup.cfg +4 -0
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
changelog:
|
|
2
|
+
exclude:
|
|
3
|
+
labels:
|
|
4
|
+
- ignore-for-release
|
|
5
|
+
authors:
|
|
6
|
+
- octocat
|
|
7
|
+
categories:
|
|
8
|
+
- title: "๐ชฒ Bug Fixes"
|
|
9
|
+
labels:
|
|
10
|
+
- bug
|
|
11
|
+
- title: "โจ New Features"
|
|
12
|
+
labels:
|
|
13
|
+
- enhancement
|
|
14
|
+
- title: "๐ Other Changes"
|
|
15
|
+
labels:
|
|
16
|
+
- "*"
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
|
|
6
|
+
jobs:
|
|
7
|
+
python-build:
|
|
8
|
+
name: "๐๏ธ Build Python wheels"
|
|
9
|
+
strategy:
|
|
10
|
+
matrix:
|
|
11
|
+
python:
|
|
12
|
+
- "3.9"
|
|
13
|
+
- "3.10"
|
|
14
|
+
- "3.11"
|
|
15
|
+
runs-on: ubuntu-latest
|
|
16
|
+
steps:
|
|
17
|
+
- name: "๐ฅ Check-out"
|
|
18
|
+
uses: actions/checkout@v4
|
|
19
|
+
- name: "๐งฐ Install Python"
|
|
20
|
+
uses: actions/setup-python@v5
|
|
21
|
+
with:
|
|
22
|
+
python-version: ${{ matrix.python }}
|
|
23
|
+
cache: pip
|
|
24
|
+
cache-dependency-path: pyproject.toml
|
|
25
|
+
- name: "๐งฐ Install dependencies"
|
|
26
|
+
run: pip install build twine
|
|
27
|
+
- name: "๐๏ธ Build Python wheels"
|
|
28
|
+
run: python -m build
|
|
29
|
+
- name: "๐งช Check package bundles"
|
|
30
|
+
run: twine check dist/*
|
|
31
|
+
- name: "๐ค Upload Python wheels"
|
|
32
|
+
uses: actions/upload-artifact@v4
|
|
33
|
+
if: matrix.python == '3.11'
|
|
34
|
+
with:
|
|
35
|
+
name: wheels
|
|
36
|
+
path: dist
|
|
37
|
+
python-code-check:
|
|
38
|
+
name: ๐ Check Python code
|
|
39
|
+
strategy:
|
|
40
|
+
matrix:
|
|
41
|
+
python:
|
|
42
|
+
- "3.9"
|
|
43
|
+
- "3.10"
|
|
44
|
+
- "3.11"
|
|
45
|
+
runs-on: ubuntu-latest
|
|
46
|
+
steps:
|
|
47
|
+
- name: ๐ฅ Check-out
|
|
48
|
+
uses: actions/checkout@v4
|
|
49
|
+
- name: ๐งฐ Install Python
|
|
50
|
+
uses: actions/setup-python@v5
|
|
51
|
+
with:
|
|
52
|
+
python-version: ${{ matrix.python }}
|
|
53
|
+
cache: pip
|
|
54
|
+
cache-dependency-path: pyproject.toml
|
|
55
|
+
- name: ๐งฐ Install dependencies
|
|
56
|
+
run: pip install .[tests]
|
|
57
|
+
- name: ๐ Check Python code
|
|
58
|
+
run: ruff check .
|
|
59
|
+
python-typing:
|
|
60
|
+
name: "๐ Check Python static typing"
|
|
61
|
+
strategy:
|
|
62
|
+
matrix:
|
|
63
|
+
python:
|
|
64
|
+
- "3.9"
|
|
65
|
+
- "3.10"
|
|
66
|
+
- "3.11"
|
|
67
|
+
runs-on: ubuntu-latest
|
|
68
|
+
steps:
|
|
69
|
+
- name: "๐ฅ Check-out"
|
|
70
|
+
uses: actions/checkout@v4
|
|
71
|
+
- name: "๐งฐ Install Protoc"
|
|
72
|
+
run: sudo apt install protobuf-compiler
|
|
73
|
+
- name: "๐งฐ Install Python"
|
|
74
|
+
uses: actions/setup-python@v5
|
|
75
|
+
with:
|
|
76
|
+
python-version: ${{ matrix.python }}
|
|
77
|
+
cache: pip
|
|
78
|
+
cache-dependency-path: pyproject.toml
|
|
79
|
+
- name: "๐งฐ Install dependencies"
|
|
80
|
+
run: pip install .[tests]
|
|
81
|
+
- name: "๐ Check Python static typing"
|
|
82
|
+
run: mypy .
|
|
83
|
+
python-security:
|
|
84
|
+
name: "๐ Check Python code security"
|
|
85
|
+
strategy:
|
|
86
|
+
matrix:
|
|
87
|
+
python:
|
|
88
|
+
- "3.9"
|
|
89
|
+
- "3.10"
|
|
90
|
+
- "3.11"
|
|
91
|
+
runs-on: ubuntu-latest
|
|
92
|
+
steps:
|
|
93
|
+
- name: "๐ฅ Check-out"
|
|
94
|
+
uses: actions/checkout@v4
|
|
95
|
+
- name: "๐งฐ Install Protoc"
|
|
96
|
+
run: sudo apt install protobuf-compiler
|
|
97
|
+
- name: "๐งฐ Install Python"
|
|
98
|
+
uses: actions/setup-python@v5
|
|
99
|
+
with:
|
|
100
|
+
python-version: ${{ matrix.python }}
|
|
101
|
+
cache: pip
|
|
102
|
+
cache-dependency-path: pyproject.toml
|
|
103
|
+
- name: "๐งฐ Install dependencies"
|
|
104
|
+
run: pip install .[tests]
|
|
105
|
+
- name: "๐ Check Python code security"
|
|
106
|
+
run: bandit -c pyproject.toml -r .
|
|
107
|
+
python-test:
|
|
108
|
+
name: "๐งช Test Python code"
|
|
109
|
+
strategy:
|
|
110
|
+
matrix:
|
|
111
|
+
python:
|
|
112
|
+
- "3.9"
|
|
113
|
+
- "3.10"
|
|
114
|
+
- "3.11"
|
|
115
|
+
runs-on: ubuntu-latest
|
|
116
|
+
steps:
|
|
117
|
+
- name: "๐ฅ Check-out"
|
|
118
|
+
uses: actions/checkout@v4
|
|
119
|
+
- name: "๐งฐ Install Python"
|
|
120
|
+
uses: actions/setup-python@v5
|
|
121
|
+
with:
|
|
122
|
+
python-version: ${{ matrix.python }}
|
|
123
|
+
cache: pip
|
|
124
|
+
cache-dependency-path: pyproject.toml
|
|
125
|
+
- name: "๐งฐ Install dependencies"
|
|
126
|
+
run: pip install .[tests]
|
|
127
|
+
- name: "๐งช Test Python code"
|
|
128
|
+
run: pytest --cov --cov-report=xml .
|
|
129
|
+
- name: "๐ค Upload coverage to Codecov"
|
|
130
|
+
uses: codecov/codecov-action@v4
|
|
131
|
+
if: matrix.python == '3.11'
|
|
132
|
+
with:
|
|
133
|
+
token: ${{ secrets.CODECOV_TOKEN }}
|
|
134
|
+
python-publish:
|
|
135
|
+
name: "๐ Publish Python wheels"
|
|
136
|
+
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
|
|
137
|
+
needs:
|
|
138
|
+
- python-build
|
|
139
|
+
- python-code-check
|
|
140
|
+
- python-typing
|
|
141
|
+
- python-security
|
|
142
|
+
- python-test
|
|
143
|
+
runs-on: ubuntu-latest
|
|
144
|
+
permissions:
|
|
145
|
+
id-token: write
|
|
146
|
+
steps:
|
|
147
|
+
- name: "๐ฅ Check-out"
|
|
148
|
+
uses: actions/checkout@v4
|
|
149
|
+
- name: "๐ฅ Download Python wheels"
|
|
150
|
+
uses: actions/download-artifact@v4
|
|
151
|
+
with:
|
|
152
|
+
name: wheels
|
|
153
|
+
path: dist
|
|
154
|
+
- name: "๐ Publish Python wheels"
|
|
155
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
156
|
+
github-release:
|
|
157
|
+
name: "๐ Create GitHub release"
|
|
158
|
+
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
|
|
159
|
+
needs:
|
|
160
|
+
- python-build
|
|
161
|
+
- python-code-check
|
|
162
|
+
- python-typing
|
|
163
|
+
- python-security
|
|
164
|
+
- python-test
|
|
165
|
+
- python-publish
|
|
166
|
+
permissions:
|
|
167
|
+
contents: write
|
|
168
|
+
runs-on: ubuntu-latest
|
|
169
|
+
steps:
|
|
170
|
+
- name: "๐ฅ Check-out"
|
|
171
|
+
uses: actions/checkout@v4
|
|
172
|
+
- name: "๐ฅ Download Python wheels"
|
|
173
|
+
uses: actions/download-artifact@v4
|
|
174
|
+
with:
|
|
175
|
+
name: wheels
|
|
176
|
+
path: dist
|
|
177
|
+
- name: "๐ Create GitHub release"
|
|
178
|
+
uses: softprops/action-gh-release@v2
|
|
179
|
+
with:
|
|
180
|
+
name: Release ${{ github.ref_name }}
|
|
181
|
+
files: dist/*
|
|
182
|
+
fail_on_unmatched_files: true
|
|
183
|
+
draft: false
|
|
184
|
+
prerelease: false
|
|
185
|
+
generate_release_notes: true
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
# Created by https://www.toptal.com/developers/gitignore/api/python,visualstudiocode,intellij
|
|
2
|
+
# Edit at https://www.toptal.com/developers/gitignore?templates=python,visualstudiocode,intellij
|
|
3
|
+
|
|
4
|
+
### Intellij ###
|
|
5
|
+
.idea
|
|
6
|
+
*.iml
|
|
7
|
+
|
|
8
|
+
### Python ###
|
|
9
|
+
# Byte-compiled / optimized / DLL files
|
|
10
|
+
__pycache__/
|
|
11
|
+
*.py[cod]
|
|
12
|
+
*$py.class
|
|
13
|
+
|
|
14
|
+
# C extensions
|
|
15
|
+
*.so
|
|
16
|
+
|
|
17
|
+
# Distribution / packaging
|
|
18
|
+
.Python
|
|
19
|
+
build/
|
|
20
|
+
develop-eggs/
|
|
21
|
+
dist/
|
|
22
|
+
downloads/
|
|
23
|
+
eggs/
|
|
24
|
+
.eggs/
|
|
25
|
+
lib/
|
|
26
|
+
lib64/
|
|
27
|
+
parts/
|
|
28
|
+
sdist/
|
|
29
|
+
var/
|
|
30
|
+
wheels/
|
|
31
|
+
share/python-wheels/
|
|
32
|
+
*.egg-info/
|
|
33
|
+
.installed.cfg
|
|
34
|
+
*.egg
|
|
35
|
+
MANIFEST
|
|
36
|
+
|
|
37
|
+
# PyInstaller
|
|
38
|
+
# Usually these files are written by a python script from a template
|
|
39
|
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
|
40
|
+
*.manifest
|
|
41
|
+
*.spec
|
|
42
|
+
|
|
43
|
+
# Installer logs
|
|
44
|
+
pip-log.txt
|
|
45
|
+
pip-delete-this-directory.txt
|
|
46
|
+
|
|
47
|
+
# Unit test / coverage reports
|
|
48
|
+
htmlcov/
|
|
49
|
+
.tox/
|
|
50
|
+
.nox/
|
|
51
|
+
.coverage
|
|
52
|
+
.coverage.*
|
|
53
|
+
.cache
|
|
54
|
+
nosetests.xml
|
|
55
|
+
coverage.xml
|
|
56
|
+
*.cover
|
|
57
|
+
*.py,cover
|
|
58
|
+
.hypothesis/
|
|
59
|
+
.pytest_cache/
|
|
60
|
+
cover/
|
|
61
|
+
|
|
62
|
+
# Translations
|
|
63
|
+
*.mo
|
|
64
|
+
*.pot
|
|
65
|
+
|
|
66
|
+
# Django stuff:
|
|
67
|
+
*.log
|
|
68
|
+
local_settings.py
|
|
69
|
+
db.sqlite3
|
|
70
|
+
db.sqlite3-journal
|
|
71
|
+
|
|
72
|
+
# Flask stuff:
|
|
73
|
+
instance/
|
|
74
|
+
.webassets-cache
|
|
75
|
+
|
|
76
|
+
# Scrapy stuff:
|
|
77
|
+
.scrapy
|
|
78
|
+
|
|
79
|
+
# Sphinx documentation
|
|
80
|
+
docs/_build/
|
|
81
|
+
|
|
82
|
+
# PyBuilder
|
|
83
|
+
.pybuilder/
|
|
84
|
+
target/
|
|
85
|
+
|
|
86
|
+
# Jupyter Notebook
|
|
87
|
+
.ipynb_checkpoints
|
|
88
|
+
|
|
89
|
+
# IPython
|
|
90
|
+
profile_default/
|
|
91
|
+
ipython_config.py
|
|
92
|
+
|
|
93
|
+
# pyenv
|
|
94
|
+
# For a library or package, you might want to ignore these files since the code is
|
|
95
|
+
# intended to run in multiple environments; otherwise, check them in:
|
|
96
|
+
# .python-version
|
|
97
|
+
|
|
98
|
+
# pipenv
|
|
99
|
+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
|
100
|
+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
|
101
|
+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
|
102
|
+
# install all needed dependencies.
|
|
103
|
+
#Pipfile.lock
|
|
104
|
+
|
|
105
|
+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
|
|
106
|
+
__pypackages__/
|
|
107
|
+
|
|
108
|
+
# Celery stuff
|
|
109
|
+
celerybeat-schedule
|
|
110
|
+
celerybeat.pid
|
|
111
|
+
|
|
112
|
+
# SageMath parsed files
|
|
113
|
+
*.sage.py
|
|
114
|
+
|
|
115
|
+
# Environments
|
|
116
|
+
.env
|
|
117
|
+
.venv
|
|
118
|
+
env/
|
|
119
|
+
venv/
|
|
120
|
+
ENV/
|
|
121
|
+
env.bak/
|
|
122
|
+
venv.bak/
|
|
123
|
+
|
|
124
|
+
# Spyder project settings
|
|
125
|
+
.spyderproject
|
|
126
|
+
.spyproject
|
|
127
|
+
|
|
128
|
+
# Rope project settings
|
|
129
|
+
.ropeproject
|
|
130
|
+
|
|
131
|
+
# mkdocs documentation
|
|
132
|
+
/site
|
|
133
|
+
|
|
134
|
+
# mypy
|
|
135
|
+
.mypy_cache/
|
|
136
|
+
.dmypy.json
|
|
137
|
+
dmypy.json
|
|
138
|
+
|
|
139
|
+
# Pyre type checker
|
|
140
|
+
.pyre/
|
|
141
|
+
|
|
142
|
+
# pytype static type analyzer
|
|
143
|
+
.pytype/
|
|
144
|
+
|
|
145
|
+
# Cython debug symbols
|
|
146
|
+
cython_debug/
|
|
147
|
+
|
|
148
|
+
### VisualStudioCode ###
|
|
149
|
+
.vscode/*
|
|
150
|
+
!.vscode/settings.json
|
|
151
|
+
!.vscode/tasks.json
|
|
152
|
+
!.vscode/launch.json
|
|
153
|
+
!.vscode/extensions.json
|
|
154
|
+
*.code-workspace
|
|
155
|
+
|
|
156
|
+
# Local History for Visual Studio Code
|
|
157
|
+
.history/
|
|
158
|
+
|
|
159
|
+
### VisualStudioCode Patch ###
|
|
160
|
+
# Ignore all local history of files
|
|
161
|
+
.history
|
|
162
|
+
.ionide
|
|
163
|
+
|
|
164
|
+
# Support for Project snippet scope
|
|
165
|
+
!.vscode/*.code-snippets
|
|
166
|
+
|
|
167
|
+
# End of https://www.toptal.com/developers/gitignore/api/python,visualstudiocode,intellij
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2024 Jan Heinrich Merker
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: ray-elasticsearch
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Unified, type-safe access to web archive APIs.
|
|
5
|
+
Author-email: Jan Heinrich Merker <heinrich.merker@uni-jena.de>
|
|
6
|
+
Project-URL: Homepage, https://github.com/heinrichreimer/ray-elasticsearch
|
|
7
|
+
Project-URL: Bug Tracker, https://github.com/heinrichreimer/ray-elasticsearch/issues
|
|
8
|
+
Classifier: Programming Language :: Python :: 3
|
|
9
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Intended Audience :: Science/Research
|
|
16
|
+
Classifier: Topic :: Scientific/Engineering
|
|
17
|
+
Requires-Python: >=3.9
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
License-File: LICENSE
|
|
20
|
+
Requires-Dist: ray[data]~=2.31
|
|
21
|
+
Requires-Dist: typing-extensions~=4.0
|
|
22
|
+
Provides-Extra: tests
|
|
23
|
+
Requires-Dist: bandit[toml]~=1.7; extra == "tests"
|
|
24
|
+
Requires-Dist: elasticsearch~=8.14; extra == "tests"
|
|
25
|
+
Requires-Dist: mypy~=1.6; extra == "tests"
|
|
26
|
+
Requires-Dist: pandas-stubs~=2.0; extra == "tests"
|
|
27
|
+
Requires-Dist: pytest~=8.0; extra == "tests"
|
|
28
|
+
Requires-Dist: pytest-cov~=5.0; extra == "tests"
|
|
29
|
+
Requires-Dist: pyarrow-stubs~=10.0; extra == "tests"
|
|
30
|
+
Requires-Dist: ruff~=0.5.0; extra == "tests"
|
|
31
|
+
|
|
32
|
+
<!-- markdownlint-disable MD041 -->
|
|
33
|
+
[](https://pypi.org/project/ray-elasticsearch/)
|
|
34
|
+
[](https://github.com/heinrichreimer/ray-elasticsearch/actions/workflows/ci.yml)
|
|
35
|
+
[](https://codecov.io/github/heinrichreimer/ray-elasticsearch/)
|
|
36
|
+
[](https://pypi.org/project/ray-elasticsearch/)
|
|
37
|
+
[](https://github.com/heinrichreimer/ray-elasticsearch/issues)
|
|
38
|
+
[](https://github.com/heinrichreimer/ray-elasticsearch/commits)
|
|
39
|
+
[](https://pypi.org/project/ray-elasticsearch/)
|
|
40
|
+
[](LICENSE)
|
|
41
|
+
|
|
42
|
+
# โ๏ธ ray-elasticsearch
|
|
43
|
+
|
|
44
|
+
Ray data source and sink for Elasticsearch.
|
|
45
|
+
|
|
46
|
+
Use this minimal library if you plan to read or write data from/to [Elasticsearch](https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html) massively parallel for data processing in [Ray](https://docs.ray.io/en/latest/data/data.html). Internally, the library uses parallelized [sliced point-in-time search](https://www.elastic.co/guide/en/elasticsearch/reference/current/point-in-time-api.html#search-slicing) for reading and parallelized [bulk requests](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html) for writing data, the two most efficient ways to read/write data to/from Elasticsearch. Note, that this library does _not_ guarantee any specific ordering of the results, though, the scores are returned.
|
|
47
|
+
|
|
48
|
+
## Installation
|
|
49
|
+
|
|
50
|
+
Install the package from PyPI:
|
|
51
|
+
|
|
52
|
+
```shell
|
|
53
|
+
pip install ray-elasticsearch
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
## Usage
|
|
57
|
+
|
|
58
|
+
This library makes use of Ray's [`Datasource`](https://docs.ray.io/en/latest/data/api/doc/ray.data.Datasource.html#ray.data.Datasource) and [`Datasink`](https://docs.ray.io/en/latest/data/api/doc/ray.data.Datasink.html#ray.data.Datasink) APIs.
|
|
59
|
+
For [reading](#read-documents), use [`ElasticsearchDatasource`](#read-documents) and, for [writing](#write-documents), use [`ElasticsearchDatasink`](#write-documents).
|
|
60
|
+
|
|
61
|
+
### Read documents
|
|
62
|
+
|
|
63
|
+
You can read results from a specified index by using an `ElasticsearchDatasource` with Ray's [`read_datasource()`](https://docs.ray.io/en/latest/data/api/doc/ray.data.read_datasource.html#ray.data.read_datasource) like so:
|
|
64
|
+
|
|
65
|
+
```python
|
|
66
|
+
from ray import init
|
|
67
|
+
from ray.data import read_datasink
|
|
68
|
+
from ray_elasticsearch import ElasticsearchDatasource
|
|
69
|
+
|
|
70
|
+
init()
|
|
71
|
+
source = ElasticsearchDslDatasource(index="test")
|
|
72
|
+
res = read_datasource(source).sum("foo")
|
|
73
|
+
print(f"Read complete. Sum: {res}")
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
Use an Elasticsearch [query](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html) to filter the results:
|
|
77
|
+
|
|
78
|
+
```python
|
|
79
|
+
source = ElasticsearchDslDatasource(
|
|
80
|
+
index="test",
|
|
81
|
+
query={
|
|
82
|
+
"match": {
|
|
83
|
+
"text": "foo bar",
|
|
84
|
+
},
|
|
85
|
+
},
|
|
86
|
+
)
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
Note that the parallel read does not enforce any ordering of the results even though the results are scored by Elasticsearch.
|
|
90
|
+
|
|
91
|
+
Normally, it is not necessary to specify a fixed concurrency level.
|
|
92
|
+
The data source will automatically determine the optimal concurrency based on the disk size of the Elasticsearch index and the Ray cluster capabilities.
|
|
93
|
+
You can, however, override the concurrency by setting the `concurrency` parameter in Ray's [`read_datasource()`](https://docs.ray.io/en/latest/data/api/doc/ray.data.read_datasource.html#ray.data.read_datasource).
|
|
94
|
+
|
|
95
|
+
### Write documents
|
|
96
|
+
|
|
97
|
+
Writing documents works similarly by using the `ElasticsearchDatasink` with Ray's [`write_datasink()`](https://docs.ray.io/en/latest/data/api/doc/ray.data.Dataset.write_datasink.html#ray.data.Dataset.write_datasink):
|
|
98
|
+
|
|
99
|
+
```python
|
|
100
|
+
from ray import init
|
|
101
|
+
from ray.data import range
|
|
102
|
+
from ray_elasticsearch import ElasticsearchDatasink
|
|
103
|
+
|
|
104
|
+
init()
|
|
105
|
+
sink = ElasticsearchDatasink(index="test")
|
|
106
|
+
range(10_000).write_datasink(sink)
|
|
107
|
+
print("Write complete.")
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
Concurrency can again be limited by specifying the `concurrency` parameter in Ray's [`write_datasink()`](https://docs.ray.io/en/latest/data/api/doc/ray.data.Dataset.write_datasink.html#ray.data.Dataset.write_datasink).
|
|
111
|
+
|
|
112
|
+
### Elasticsearch connection
|
|
113
|
+
|
|
114
|
+
Per default, the data source and sink access Elasticsearch on `localhost:9200`.
|
|
115
|
+
However, in most cases, you would instead want to continue to some remote Elasticsearch instance.
|
|
116
|
+
To do so, specify the client like in the example below, and use the same parameters as in the [`Elasticsearch()`](https://elasticsearch-py.readthedocs.io/en/latest/api/elasticsearch.html#elasticsearch.Elasticsearch) constructor:
|
|
117
|
+
|
|
118
|
+
```python
|
|
119
|
+
source = ElasticsearchDatasource(
|
|
120
|
+
index="test",
|
|
121
|
+
client_kwargs=dict(
|
|
122
|
+
hosts="<HOST>",
|
|
123
|
+
http_auth=("<USERNAME>", "<PASSWORD>"),
|
|
124
|
+
max_retries=10,
|
|
125
|
+
),
|
|
126
|
+
)
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
For the full list of allowed arguments in the `client_kwargs` dictionary, refer to the documentation of the [`Elasticsearch()`](https://elasticsearch-py.readthedocs.io/en/latest/api/elasticsearch.html#elasticsearch.Elasticsearch) constructor.
|
|
130
|
+
|
|
131
|
+
### Elasticsearch DSL
|
|
132
|
+
|
|
133
|
+
To simplify query construction, you can also use the [Elasticsearch DSL](https://elasticsearch-dsl.readthedocs.io/en/latest/) and its corresponding data source (`ElasticsearchDslDatasource`) and sink (`ElasticsearchDslDatasink`):
|
|
134
|
+
|
|
135
|
+
```python
|
|
136
|
+
from elasticsearch7_dsl import Document
|
|
137
|
+
from elasticsearch7_dsl.query import Exists
|
|
138
|
+
from ray_elasticsearch import ElasticsearchDslDatasource, ElasticsearchDslDatasink
|
|
139
|
+
|
|
140
|
+
class Foo(Document):
|
|
141
|
+
class Index:
|
|
142
|
+
name = "test_foo"
|
|
143
|
+
text: str = Text()
|
|
144
|
+
|
|
145
|
+
source = ElasticsearchDslDatasource(
|
|
146
|
+
index=Foo,
|
|
147
|
+
query=Exists(field="doi"),
|
|
148
|
+
)
|
|
149
|
+
sink = ElasticsearchDslDatasink(index=Foo)
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
Note that, unlike in [Elasticsearch DSL](https://elasticsearch-dsl.readthedocs.io/en/latest/), the results are not parsed as Python objects but instead remain Python dictionaries, due to Ray internally transforming everything in [Arrow format](https://arrow.apache.org/docs/python/index.html).
|
|
153
|
+
|
|
154
|
+
## Development
|
|
155
|
+
|
|
156
|
+
To build this package and contribute to its development you need to install the `build`, `setuptools` and `wheel` packages:
|
|
157
|
+
|
|
158
|
+
```shell
|
|
159
|
+
pip install build setuptools wheel
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
(On most systems, these packages are already pre-installed.)
|
|
163
|
+
|
|
164
|
+
### Development installation
|
|
165
|
+
|
|
166
|
+
Install package and test dependencies:
|
|
167
|
+
|
|
168
|
+
```shell
|
|
169
|
+
pip install -e .[tests]
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
### Testing
|
|
173
|
+
|
|
174
|
+
Verify your changes against the test suite to verify.
|
|
175
|
+
|
|
176
|
+
```shell
|
|
177
|
+
ruff check . # Code format and LINT
|
|
178
|
+
mypy . # Static typing
|
|
179
|
+
bandit -c pyproject.toml -r . # Security
|
|
180
|
+
pytest . # Unit tests
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
Please also add tests for your newly developed code.
|
|
184
|
+
|
|
185
|
+
### Build wheels
|
|
186
|
+
|
|
187
|
+
Wheels for this package can be built with:
|
|
188
|
+
|
|
189
|
+
```shell
|
|
190
|
+
python -m build
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
## Support
|
|
194
|
+
|
|
195
|
+
If you have any problems using this package, please file an [issue](https://github.com/heinrichreimer/ray-elasticsearch/issues/new).
|
|
196
|
+
We're happy to help!
|
|
197
|
+
|
|
198
|
+
## License
|
|
199
|
+
|
|
200
|
+
This repository is released under the [MIT license](LICENSE).
|