splent-feature-elasticsearch 0.1.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- splent_feature_elasticsearch-0.1.2/MANIFEST.in +24 -0
- splent_feature_elasticsearch-0.1.2/PKG-INFO +111 -0
- splent_feature_elasticsearch-0.1.2/README.md +104 -0
- splent_feature_elasticsearch-0.1.2/pyproject.toml +83 -0
- splent_feature_elasticsearch-0.1.2/setup.cfg +4 -0
- splent_feature_elasticsearch-0.1.2/src/splent_feature_elasticsearch.egg-info/PKG-INFO +111 -0
- splent_feature_elasticsearch-0.1.2/src/splent_feature_elasticsearch.egg-info/SOURCES.txt +16 -0
- splent_feature_elasticsearch-0.1.2/src/splent_feature_elasticsearch.egg-info/dependency_links.txt +1 -0
- splent_feature_elasticsearch-0.1.2/src/splent_feature_elasticsearch.egg-info/requires.txt +1 -0
- splent_feature_elasticsearch-0.1.2/src/splent_feature_elasticsearch.egg-info/top_level.txt +1 -0
- splent_feature_elasticsearch-0.1.2/src/splent_io/splent_feature_elasticsearch/__init__.py +14 -0
- splent_feature_elasticsearch-0.1.2/src/splent_io/splent_feature_elasticsearch/commands.py +40 -0
- splent_feature_elasticsearch-0.1.2/src/splent_io/splent_feature_elasticsearch/config.py +46 -0
- splent_feature_elasticsearch-0.1.2/src/splent_io/splent_feature_elasticsearch/services.py +469 -0
- splent_feature_elasticsearch-0.1.2/src/splent_io/splent_feature_elasticsearch/signals.py +19 -0
- splent_feature_elasticsearch-0.1.2/src/splent_io/splent_feature_elasticsearch/translations/es/LC_MESSAGES/messages.mo +0 -0
- splent_feature_elasticsearch-0.1.2/src/splent_io/splent_feature_elasticsearch/translations/es/LC_MESSAGES/messages.po +20 -0
- splent_feature_elasticsearch-0.1.2/src/splent_io/splent_feature_elasticsearch/translations/messages.pot +19 -0
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# Frontend assets: the whole tree, not only the compiled bundles.
|
|
2
|
+
#
|
|
3
|
+
# A feature that registers a plain stylesheet with register_asset("css", …)
|
|
4
|
+
# serves it straight from assets/css, and a vendored library lives under
|
|
5
|
+
# assets/vendor. Shipping assets/dist alone put no stylesheet at all in the
|
|
6
|
+
# built package, so every production image came up without its theme, its
|
|
7
|
+
# skin and its editor, while development looked fine because that path
|
|
8
|
+
# installs the checkout in place and reads the files off disk.
|
|
9
|
+
recursive-include src/splent_io/splent_feature_elasticsearch/assets *.css *.js *.map *.json
|
|
10
|
+
recursive-include src/splent_io/splent_feature_elasticsearch/assets *.svg *.png *.jpg *.jpeg *.gif *.webp *.ico
|
|
11
|
+
recursive-include src/splent_io/splent_feature_elasticsearch/assets *.woff *.woff2 *.ttf *.eot
|
|
12
|
+
|
|
13
|
+
# Build inputs, never shipped.
|
|
14
|
+
prune src/splent_io/splent_feature_elasticsearch/assets/node_modules
|
|
15
|
+
|
|
16
|
+
# Jinja templates (feature views + hook fragments)
|
|
17
|
+
recursive-include src/splent_io/splent_feature_elasticsearch/templates *.html
|
|
18
|
+
|
|
19
|
+
# Alembic migration config and scripts
|
|
20
|
+
recursive-include src/splent_io/splent_feature_elasticsearch/migrations *.py *.ini *.mako
|
|
21
|
+
|
|
22
|
+
# Translations. The .mo is what gettext reads at runtime; the .po
|
|
23
|
+
# travels with it so the next translator has something to edit.
|
|
24
|
+
recursive-include src/splent_io/splent_feature_elasticsearch/translations *.mo *.po *.pot
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: splent_feature_elasticsearch
|
|
3
|
+
Version: 0.1.2
|
|
4
|
+
Requires-Python: >=3.13
|
|
5
|
+
Description-Content-Type: text/markdown
|
|
6
|
+
Requires-Dist: elasticsearch<9,>=8
|
|
7
|
+
|
|
8
|
+
# splent_feature_elasticsearch
|
|
9
|
+
|
|
10
|
+
Reusable Elasticsearch infrastructure for SPLENT products (service archetype).
|
|
11
|
+
Ships a single-node Elasticsearch 8.x container and a generic, domain-agnostic
|
|
12
|
+
`ElasticsearchService` that any feature can consume through the service locator.
|
|
13
|
+
|
|
14
|
+
## What it provides
|
|
15
|
+
|
|
16
|
+
- `ElasticsearchService` (registered in the service locator):
|
|
17
|
+
`available()`, `index_name(name)`, `ensure_index(name, mappings=None)`,
|
|
18
|
+
`index_document(index, id, body)`, `bulk_index(index, documents)`,
|
|
19
|
+
`delete_document(index, id)`, `refresh_index(name)`,
|
|
20
|
+
`search(index, query_body)`, `delete_index(name)`, `health()`.
|
|
21
|
+
- Docker: `docker/docker-compose.dev.yml` (host port published) and
|
|
22
|
+
`docker/docker-compose.prod.yml` (internal-only on `splent_network`),
|
|
23
|
+
both with a healthcheck and a named `elasticsearch_data` volume.
|
|
24
|
+
- CLI: `splent feature:elasticsearch status`.
|
|
25
|
+
|
|
26
|
+
## One node, many products
|
|
27
|
+
|
|
28
|
+
In development the CLI starts each feature as its own compose project, named
|
|
29
|
+
after the feature reference with no product name in it. Every product in the
|
|
30
|
+
workspace therefore talks to the same Elasticsearch container and writes into
|
|
31
|
+
the same data volume. Index names are a flat, node-wide namespace, so two
|
|
32
|
+
products that both index `pages` would overwrite each other's documents and
|
|
33
|
+
answer searches with each other's material.
|
|
34
|
+
|
|
35
|
+
Every index name handed to the service is prefixed with
|
|
36
|
+
`ELASTICSEARCH_INDEX_PREFIX`, whose shipped value `__PRODUCT__` is resolved to
|
|
37
|
+
the product name by `product:env --merge` and `product:build`. A feature keeps
|
|
38
|
+
using its own short name and never has to know.
|
|
39
|
+
|
|
40
|
+
```python
|
|
41
|
+
es.ensure_index("pages") # creates innosoft_app_pages
|
|
42
|
+
es.index_name("pages") # "innosoft_app_pages"
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
`splent feature:elasticsearch status` prints the namespace in use, which is the
|
|
46
|
+
quickest way to check that a product is not sharing indices with its neighbour.
|
|
47
|
+
|
|
48
|
+
An index is a cache and never the authority on visibility. This service stores
|
|
49
|
+
and returns whatever documents it is given; whether a particular reader may see
|
|
50
|
+
a particular hit is decided at request time by the feature that owns the
|
|
51
|
+
material.
|
|
52
|
+
|
|
53
|
+
## Graceful degradation and typed errors
|
|
54
|
+
|
|
55
|
+
Elasticsearch is a soft dependency. `available()` never raises, and every other
|
|
56
|
+
operation raises `ElasticsearchError`, never a raw client exception. Two
|
|
57
|
+
subclasses say what happened and what to do about it.
|
|
58
|
+
|
|
59
|
+
| Exception | Meaning | Usual response |
|
|
60
|
+
| --- | --- | --- |
|
|
61
|
+
| `ElasticsearchUnavailable` | Disabled, or the node could not be reached. | Fall back to a slower search. |
|
|
62
|
+
| `ElasticsearchRequestError` | The node answered and refused the request (malformed query, rejected mapping, missing permission). | Let it surface. Retrying will not help. |
|
|
63
|
+
|
|
64
|
+
```python
|
|
65
|
+
from splent_framework.services.service_locator import service_proxy
|
|
66
|
+
from splent_io.splent_feature_elasticsearch.services import ElasticsearchUnavailable
|
|
67
|
+
|
|
68
|
+
es = service_proxy("ElasticsearchService")
|
|
69
|
+
|
|
70
|
+
try:
|
|
71
|
+
results = es.search("posts", {"query": {"match": {"title": query}}})
|
|
72
|
+
except ElasticsearchUnavailable:
|
|
73
|
+
results = fallback_sql_search(query)
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
## Bulk indexing
|
|
77
|
+
|
|
78
|
+
A full reindex is thousands of documents, and one HTTP round trip each turns a
|
|
79
|
+
maintenance command into something nobody runs. `bulk_index` consumes any
|
|
80
|
+
iterable lazily, sends batches through the `_bulk` API and reports what
|
|
81
|
+
happened, so a single bad document does not cost you the other 9999.
|
|
82
|
+
|
|
83
|
+
```python
|
|
84
|
+
report = es.bulk_index(
|
|
85
|
+
"pages",
|
|
86
|
+
({"id": page.id, "body": {"title": page.title, "body": page.text}} for page in pages),
|
|
87
|
+
)
|
|
88
|
+
es.refresh_index("pages")
|
|
89
|
+
# {"indexed": 9999, "failed": 1, "errors": [{"id": 42, "error": "..."}]}
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
Only a failure of the request itself raises. Per-document failures come back in
|
|
93
|
+
the report, with the error list capped so a broken mapping cannot flood a log.
|
|
94
|
+
|
|
95
|
+
## Configuration
|
|
96
|
+
|
|
97
|
+
| Env var | Default | Purpose |
|
|
98
|
+
| --- | --- | --- |
|
|
99
|
+
| `ELASTICSEARCH_URL` | `http://localhost:9200` | Node URL. The Docker `.env` sets `http://splent_feature_elasticsearch:9200` for container-to-container access. |
|
|
100
|
+
| `ELASTICSEARCH_ENABLED` | `true` | Feature kill switch. When false all operations degrade immediately. |
|
|
101
|
+
| `ELASTICSEARCH_INDEX_PREFIX` | `splent` | Namespace for this product's indices. The Docker `.env` ships `__PRODUCT__`, resolved to the product name. |
|
|
102
|
+
| `ELASTICSEARCH_HOST_PORT` | `9201` | Host port published in development only. |
|
|
103
|
+
| `ELASTICSEARCH_JAVA_OPTS` | `-Xms512m -Xmx512m` | Heap for the node, read by the production compose file. |
|
|
104
|
+
|
|
105
|
+
## Tests
|
|
106
|
+
|
|
107
|
+
```bash
|
|
108
|
+
splent feature:test splent_feature_elasticsearch
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
Unit tests mock the client boundary, so no network access is required.
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
# splent_feature_elasticsearch
|
|
2
|
+
|
|
3
|
+
Reusable Elasticsearch infrastructure for SPLENT products (service archetype).
|
|
4
|
+
Ships a single-node Elasticsearch 8.x container and a generic, domain-agnostic
|
|
5
|
+
`ElasticsearchService` that any feature can consume through the service locator.
|
|
6
|
+
|
|
7
|
+
## What it provides
|
|
8
|
+
|
|
9
|
+
- `ElasticsearchService` (registered in the service locator):
|
|
10
|
+
`available()`, `index_name(name)`, `ensure_index(name, mappings=None)`,
|
|
11
|
+
`index_document(index, id, body)`, `bulk_index(index, documents)`,
|
|
12
|
+
`delete_document(index, id)`, `refresh_index(name)`,
|
|
13
|
+
`search(index, query_body)`, `delete_index(name)`, `health()`.
|
|
14
|
+
- Docker: `docker/docker-compose.dev.yml` (host port published) and
|
|
15
|
+
`docker/docker-compose.prod.yml` (internal-only on `splent_network`),
|
|
16
|
+
both with a healthcheck and a named `elasticsearch_data` volume.
|
|
17
|
+
- CLI: `splent feature:elasticsearch status`.
|
|
18
|
+
|
|
19
|
+
## One node, many products
|
|
20
|
+
|
|
21
|
+
In development the CLI starts each feature as its own compose project, named
|
|
22
|
+
after the feature reference with no product name in it. Every product in the
|
|
23
|
+
workspace therefore talks to the same Elasticsearch container and writes into
|
|
24
|
+
the same data volume. Index names are a flat, node-wide namespace, so two
|
|
25
|
+
products that both index `pages` would overwrite each other's documents and
|
|
26
|
+
answer searches with each other's material.
|
|
27
|
+
|
|
28
|
+
Every index name handed to the service is prefixed with
|
|
29
|
+
`ELASTICSEARCH_INDEX_PREFIX`, whose shipped value `__PRODUCT__` is resolved to
|
|
30
|
+
the product name by `product:env --merge` and `product:build`. A feature keeps
|
|
31
|
+
using its own short name and never has to know.
|
|
32
|
+
|
|
33
|
+
```python
|
|
34
|
+
es.ensure_index("pages") # creates innosoft_app_pages
|
|
35
|
+
es.index_name("pages") # "innosoft_app_pages"
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
`splent feature:elasticsearch status` prints the namespace in use, which is the
|
|
39
|
+
quickest way to check that a product is not sharing indices with its neighbour.
|
|
40
|
+
|
|
41
|
+
An index is a cache and never the authority on visibility. This service stores
|
|
42
|
+
and returns whatever documents it is given; whether a particular reader may see
|
|
43
|
+
a particular hit is decided at request time by the feature that owns the
|
|
44
|
+
material.
|
|
45
|
+
|
|
46
|
+
## Graceful degradation and typed errors
|
|
47
|
+
|
|
48
|
+
Elasticsearch is a soft dependency. `available()` never raises, and every other
|
|
49
|
+
operation raises `ElasticsearchError`, never a raw client exception. Two
|
|
50
|
+
subclasses say what happened and what to do about it.
|
|
51
|
+
|
|
52
|
+
| Exception | Meaning | Usual response |
|
|
53
|
+
| --- | --- | --- |
|
|
54
|
+
| `ElasticsearchUnavailable` | Disabled, or the node could not be reached. | Fall back to a slower search. |
|
|
55
|
+
| `ElasticsearchRequestError` | The node answered and refused the request (malformed query, rejected mapping, missing permission). | Let it surface. Retrying will not help. |
|
|
56
|
+
|
|
57
|
+
```python
|
|
58
|
+
from splent_framework.services.service_locator import service_proxy
|
|
59
|
+
from splent_io.splent_feature_elasticsearch.services import ElasticsearchUnavailable
|
|
60
|
+
|
|
61
|
+
es = service_proxy("ElasticsearchService")
|
|
62
|
+
|
|
63
|
+
try:
|
|
64
|
+
results = es.search("posts", {"query": {"match": {"title": query}}})
|
|
65
|
+
except ElasticsearchUnavailable:
|
|
66
|
+
results = fallback_sql_search(query)
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
## Bulk indexing
|
|
70
|
+
|
|
71
|
+
A full reindex is thousands of documents, and one HTTP round trip each turns a
|
|
72
|
+
maintenance command into something nobody runs. `bulk_index` consumes any
|
|
73
|
+
iterable lazily, sends batches through the `_bulk` API and reports what
|
|
74
|
+
happened, so a single bad document does not cost you the other 9999.
|
|
75
|
+
|
|
76
|
+
```python
|
|
77
|
+
report = es.bulk_index(
|
|
78
|
+
"pages",
|
|
79
|
+
({"id": page.id, "body": {"title": page.title, "body": page.text}} for page in pages),
|
|
80
|
+
)
|
|
81
|
+
es.refresh_index("pages")
|
|
82
|
+
# {"indexed": 9999, "failed": 1, "errors": [{"id": 42, "error": "..."}]}
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
Only a failure of the request itself raises. Per-document failures come back in
|
|
86
|
+
the report, with the error list capped so a broken mapping cannot flood a log.
|
|
87
|
+
|
|
88
|
+
## Configuration
|
|
89
|
+
|
|
90
|
+
| Env var | Default | Purpose |
|
|
91
|
+
| --- | --- | --- |
|
|
92
|
+
| `ELASTICSEARCH_URL` | `http://localhost:9200` | Node URL. The Docker `.env` sets `http://splent_feature_elasticsearch:9200` for container-to-container access. |
|
|
93
|
+
| `ELASTICSEARCH_ENABLED` | `true` | Feature kill switch. When false all operations degrade immediately. |
|
|
94
|
+
| `ELASTICSEARCH_INDEX_PREFIX` | `splent` | Namespace for this product's indices. The Docker `.env` ships `__PRODUCT__`, resolved to the product name. |
|
|
95
|
+
| `ELASTICSEARCH_HOST_PORT` | `9201` | Host port published in development only. |
|
|
96
|
+
| `ELASTICSEARCH_JAVA_OPTS` | `-Xms512m -Xmx512m` | Heap for the node, read by the production compose file. |
|
|
97
|
+
|
|
98
|
+
## Tests
|
|
99
|
+
|
|
100
|
+
```bash
|
|
101
|
+
splent feature:test splent_feature_elasticsearch
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
Unit tests mock the client boundary, so no network access is required.
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=80.3.1", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "splent_feature_elasticsearch"
|
|
7
|
+
version = "0.1.2"
|
|
8
|
+
readme = "README.md"
|
|
9
|
+
requires-python = ">=3.13"
|
|
10
|
+
|
|
11
|
+
dependencies = [
|
|
12
|
+
"elasticsearch>=8,<9",
|
|
13
|
+
]
|
|
14
|
+
|
|
15
|
+
[tool.setuptools]
|
|
16
|
+
package-dir = { "" = "src" }
|
|
17
|
+
include-package-data = true
|
|
18
|
+
|
|
19
|
+
[tool.setuptools.packages.find]
|
|
20
|
+
where = ["src"]
|
|
21
|
+
exclude = ["*.tests", "*.tests.*"]
|
|
22
|
+
|
|
23
|
+
[tool.splent]
|
|
24
|
+
cli_version = "1.14.0"
|
|
25
|
+
|
|
26
|
+
# ── Feature Contract (auto-generated) ────────────────────────────────────────
|
|
27
|
+
# Do not edit manually — re-run `splent feature:contract --write` to refresh.
|
|
28
|
+
# Preserved across refreshes: description, category, tags, env,
|
|
29
|
+
# requires.features_manual (dependencies invisible to static analysis)
|
|
30
|
+
# and provides.routes_manual (routes built from configuration at
|
|
31
|
+
# registration time, which have no literal path to find in source).
|
|
32
|
+
[tool.splent.contract]
|
|
33
|
+
description = "Generic Elasticsearch infrastructure with an ElasticsearchService (index/search) that degrades gracefully"
|
|
34
|
+
archetype = "service"
|
|
35
|
+
|
|
36
|
+
[tool.splent.contract.provides]
|
|
37
|
+
routes = []
|
|
38
|
+
blueprints = []
|
|
39
|
+
models = []
|
|
40
|
+
commands = ["status"]
|
|
41
|
+
hooks = []
|
|
42
|
+
services = ["ElasticsearchService"]
|
|
43
|
+
signals = []
|
|
44
|
+
translations = ["es"]
|
|
45
|
+
docker = ["docker-compose.dev.yml", "docker-compose.prod.yml"]
|
|
46
|
+
|
|
47
|
+
[tool.splent.contract.requires]
|
|
48
|
+
features = []
|
|
49
|
+
env_vars = ["ELASTICSEARCH_ENABLED", "ELASTICSEARCH_INDEX_PREFIX", "SPLENT_APP"]
|
|
50
|
+
signals = []
|
|
51
|
+
|
|
52
|
+
[tool.splent.contract.extensible]
|
|
53
|
+
services = ["ElasticsearchService"]
|
|
54
|
+
templates = []
|
|
55
|
+
models = []
|
|
56
|
+
hooks = []
|
|
57
|
+
routes = false
|
|
58
|
+
|
|
59
|
+
[tool.splent.contract.docker]
|
|
60
|
+
services = ["splent_feature_elasticsearch"]
|
|
61
|
+
ports = ["${ELASTICSEARCH_HOST_PORT}:9200"]
|
|
62
|
+
volumes = ["elasticsearch_data"]
|
|
63
|
+
networks = ["splent_network"]
|
|
64
|
+
build = false
|
|
65
|
+
healthcheck = true
|
|
66
|
+
|
|
67
|
+
[tool.splent.contract.docker.depends_on]
|
|
68
|
+
services = []
|
|
69
|
+
|
|
70
|
+
# ── Configuration ─────────────────────────────────────────────────────────
|
|
71
|
+
# ELASTICSEARCH_URL is not here on purpose: its value names the feature's
|
|
72
|
+
# container, and a connection string that points at infrastructure lives in
|
|
73
|
+
# docker/.env.example next to the compose files that define it.
|
|
74
|
+
[tool.splent.config]
|
|
75
|
+
# Kill switch. False makes every operation degrade immediately without ever
|
|
76
|
+
# opening a connection, which is how a deployment runs without a search node.
|
|
77
|
+
ELASTICSEARCH_ENABLED = true
|
|
78
|
+
# Namespace for this product's indices. __PRODUCT__ is resolved to the
|
|
79
|
+
# product name by product:env --merge and product:build; an empty value
|
|
80
|
+
# makes the service fall back to the running product's name, which is the
|
|
81
|
+
# same name. It matters whenever a product is pointed at a node it did not
|
|
82
|
+
# start, because index names are a node-wide namespace.
|
|
83
|
+
ELASTICSEARCH_INDEX_PREFIX = "__PRODUCT__"
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: splent_feature_elasticsearch
|
|
3
|
+
Version: 0.1.2
|
|
4
|
+
Requires-Python: >=3.13
|
|
5
|
+
Description-Content-Type: text/markdown
|
|
6
|
+
Requires-Dist: elasticsearch<9,>=8
|
|
7
|
+
|
|
8
|
+
# splent_feature_elasticsearch
|
|
9
|
+
|
|
10
|
+
Reusable Elasticsearch infrastructure for SPLENT products (service archetype).
|
|
11
|
+
Ships a single-node Elasticsearch 8.x container and a generic, domain-agnostic
|
|
12
|
+
`ElasticsearchService` that any feature can consume through the service locator.
|
|
13
|
+
|
|
14
|
+
## What it provides
|
|
15
|
+
|
|
16
|
+
- `ElasticsearchService` (registered in the service locator):
|
|
17
|
+
`available()`, `index_name(name)`, `ensure_index(name, mappings=None)`,
|
|
18
|
+
`index_document(index, id, body)`, `bulk_index(index, documents)`,
|
|
19
|
+
`delete_document(index, id)`, `refresh_index(name)`,
|
|
20
|
+
`search(index, query_body)`, `delete_index(name)`, `health()`.
|
|
21
|
+
- Docker: `docker/docker-compose.dev.yml` (host port published) and
|
|
22
|
+
`docker/docker-compose.prod.yml` (internal-only on `splent_network`),
|
|
23
|
+
both with a healthcheck and a named `elasticsearch_data` volume.
|
|
24
|
+
- CLI: `splent feature:elasticsearch status`.
|
|
25
|
+
|
|
26
|
+
## One node, many products
|
|
27
|
+
|
|
28
|
+
In development the CLI starts each feature as its own compose project, named
|
|
29
|
+
after the feature reference with no product name in it. Every product in the
|
|
30
|
+
workspace therefore talks to the same Elasticsearch container and writes into
|
|
31
|
+
the same data volume. Index names are a flat, node-wide namespace, so two
|
|
32
|
+
products that both index `pages` would overwrite each other's documents and
|
|
33
|
+
answer searches with each other's material.
|
|
34
|
+
|
|
35
|
+
Every index name handed to the service is prefixed with
|
|
36
|
+
`ELASTICSEARCH_INDEX_PREFIX`, whose shipped value `__PRODUCT__` is resolved to
|
|
37
|
+
the product name by `product:env --merge` and `product:build`. A feature keeps
|
|
38
|
+
using its own short name and never has to know.
|
|
39
|
+
|
|
40
|
+
```python
|
|
41
|
+
es.ensure_index("pages") # creates innosoft_app_pages
|
|
42
|
+
es.index_name("pages") # "innosoft_app_pages"
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
`splent feature:elasticsearch status` prints the namespace in use, which is the
|
|
46
|
+
quickest way to check that a product is not sharing indices with its neighbour.
|
|
47
|
+
|
|
48
|
+
An index is a cache and never the authority on visibility. This service stores
|
|
49
|
+
and returns whatever documents it is given; whether a particular reader may see
|
|
50
|
+
a particular hit is decided at request time by the feature that owns the
|
|
51
|
+
material.
|
|
52
|
+
|
|
53
|
+
## Graceful degradation and typed errors
|
|
54
|
+
|
|
55
|
+
Elasticsearch is a soft dependency. `available()` never raises, and every other
|
|
56
|
+
operation raises `ElasticsearchError`, never a raw client exception. Two
|
|
57
|
+
subclasses say what happened and what to do about it.
|
|
58
|
+
|
|
59
|
+
| Exception | Meaning | Usual response |
|
|
60
|
+
| --- | --- | --- |
|
|
61
|
+
| `ElasticsearchUnavailable` | Disabled, or the node could not be reached. | Fall back to a slower search. |
|
|
62
|
+
| `ElasticsearchRequestError` | The node answered and refused the request (malformed query, rejected mapping, missing permission). | Let it surface. Retrying will not help. |
|
|
63
|
+
|
|
64
|
+
```python
|
|
65
|
+
from splent_framework.services.service_locator import service_proxy
|
|
66
|
+
from splent_io.splent_feature_elasticsearch.services import ElasticsearchUnavailable
|
|
67
|
+
|
|
68
|
+
es = service_proxy("ElasticsearchService")
|
|
69
|
+
|
|
70
|
+
try:
|
|
71
|
+
results = es.search("posts", {"query": {"match": {"title": query}}})
|
|
72
|
+
except ElasticsearchUnavailable:
|
|
73
|
+
results = fallback_sql_search(query)
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
## Bulk indexing
|
|
77
|
+
|
|
78
|
+
A full reindex is thousands of documents, and one HTTP round trip each turns a
|
|
79
|
+
maintenance command into something nobody runs. `bulk_index` consumes any
|
|
80
|
+
iterable lazily, sends batches through the `_bulk` API and reports what
|
|
81
|
+
happened, so a single bad document does not cost you the other 9999.
|
|
82
|
+
|
|
83
|
+
```python
|
|
84
|
+
report = es.bulk_index(
|
|
85
|
+
"pages",
|
|
86
|
+
({"id": page.id, "body": {"title": page.title, "body": page.text}} for page in pages),
|
|
87
|
+
)
|
|
88
|
+
es.refresh_index("pages")
|
|
89
|
+
# {"indexed": 9999, "failed": 1, "errors": [{"id": 42, "error": "..."}]}
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
Only a failure of the request itself raises. Per-document failures come back in
|
|
93
|
+
the report, with the error list capped so a broken mapping cannot flood a log.
|
|
94
|
+
|
|
95
|
+
## Configuration
|
|
96
|
+
|
|
97
|
+
| Env var | Default | Purpose |
|
|
98
|
+
| --- | --- | --- |
|
|
99
|
+
| `ELASTICSEARCH_URL` | `http://localhost:9200` | Node URL. The Docker `.env` sets `http://splent_feature_elasticsearch:9200` for container-to-container access. |
|
|
100
|
+
| `ELASTICSEARCH_ENABLED` | `true` | Feature kill switch. When false all operations degrade immediately. |
|
|
101
|
+
| `ELASTICSEARCH_INDEX_PREFIX` | `splent` | Namespace for this product's indices. The Docker `.env` ships `__PRODUCT__`, resolved to the product name. |
|
|
102
|
+
| `ELASTICSEARCH_HOST_PORT` | `9201` | Host port published in development only. |
|
|
103
|
+
| `ELASTICSEARCH_JAVA_OPTS` | `-Xms512m -Xmx512m` | Heap for the node, read by the production compose file. |
|
|
104
|
+
|
|
105
|
+
## Tests
|
|
106
|
+
|
|
107
|
+
```bash
|
|
108
|
+
splent feature:test splent_feature_elasticsearch
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
Unit tests mock the client boundary, so no network access is required.
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
MANIFEST.in
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
src/splent_feature_elasticsearch.egg-info/PKG-INFO
|
|
5
|
+
src/splent_feature_elasticsearch.egg-info/SOURCES.txt
|
|
6
|
+
src/splent_feature_elasticsearch.egg-info/dependency_links.txt
|
|
7
|
+
src/splent_feature_elasticsearch.egg-info/requires.txt
|
|
8
|
+
src/splent_feature_elasticsearch.egg-info/top_level.txt
|
|
9
|
+
src/splent_io/splent_feature_elasticsearch/__init__.py
|
|
10
|
+
src/splent_io/splent_feature_elasticsearch/commands.py
|
|
11
|
+
src/splent_io/splent_feature_elasticsearch/config.py
|
|
12
|
+
src/splent_io/splent_feature_elasticsearch/services.py
|
|
13
|
+
src/splent_io/splent_feature_elasticsearch/signals.py
|
|
14
|
+
src/splent_io/splent_feature_elasticsearch/translations/messages.pot
|
|
15
|
+
src/splent_io/splent_feature_elasticsearch/translations/es/LC_MESSAGES/messages.mo
|
|
16
|
+
src/splent_io/splent_feature_elasticsearch/translations/es/LC_MESSAGES/messages.po
|
splent_feature_elasticsearch-0.1.2/src/splent_feature_elasticsearch.egg-info/dependency_links.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
elasticsearch<9,>=8
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
splent_io
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
from splent_framework.blueprints.base_blueprint import create_blueprint
|
|
2
|
+
from splent_framework.services.service_locator import register_service
|
|
3
|
+
|
|
4
|
+
from splent_io.splent_feature_elasticsearch.services import ElasticsearchService
|
|
5
|
+
|
|
6
|
+
elasticsearch_bp = create_blueprint(__name__)
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def init_feature(app):
|
|
10
|
+
register_service(app, "ElasticsearchService", ElasticsearchService)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def inject_context_vars(app):
|
|
14
|
+
return {}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"""
|
|
2
|
+
CLI commands contributed by splent_feature_elasticsearch.
|
|
3
|
+
|
|
4
|
+
These commands are auto-discovered by the framework and exposed in the
|
|
5
|
+
SPLENT CLI under the ``feature:elasticsearch`` group.
|
|
6
|
+
|
|
7
|
+
Usage::
|
|
8
|
+
|
|
9
|
+
splent feature:elasticsearch status
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
import click
|
|
13
|
+
|
|
14
|
+
from splent_io.splent_feature_elasticsearch.services import ElasticsearchService
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@click.command("status")
|
|
18
|
+
def status():
|
|
19
|
+
"""Show the Elasticsearch connection status for the active product."""
|
|
20
|
+
service = ElasticsearchService()
|
|
21
|
+
info = service.health()
|
|
22
|
+
|
|
23
|
+
click.echo(f" URL: {info['url']}")
|
|
24
|
+
click.echo(f" Enabled: {'yes' if info['enabled'] else 'no'}")
|
|
25
|
+
# The node is shared by every product in the workspace, so the prefix is
|
|
26
|
+
# the only thing telling one product's indices from another's.
|
|
27
|
+
click.echo(f" Indices: {info['index_prefix']}_*")
|
|
28
|
+
|
|
29
|
+
if not info["enabled"]:
|
|
30
|
+
click.secho(" Status: disabled (ELASTICSEARCH_ENABLED=false)", fg="yellow")
|
|
31
|
+
return
|
|
32
|
+
|
|
33
|
+
if info["available"]:
|
|
34
|
+
click.secho(" Status: available", fg="green")
|
|
35
|
+
else:
|
|
36
|
+
click.secho(" Status: unreachable", fg="red")
|
|
37
|
+
raise SystemExit(1)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
cli_commands = [status]
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"""elasticsearch feature configuration.
|
|
2
|
+
|
|
3
|
+
Injects the Elasticsearch connection settings into Flask app.config.
|
|
4
|
+
|
|
5
|
+
The default URL targets a local node (http://localhost:9200). When the
|
|
6
|
+
feature's Docker service is used, the product's .env sets
|
|
7
|
+
ELASTICSEARCH_URL=http://splent_feature_elasticsearch:9200 so containers
|
|
8
|
+
resolve the node by service name on splent_network (same pattern as
|
|
9
|
+
splent_feature_redis).
|
|
10
|
+
|
|
11
|
+
ELASTICSEARCH_INDEX_PREFIX is what keeps two products apart on that node. In
|
|
12
|
+
development every product in the workspace reaches the same container and the
|
|
13
|
+
same data volume, because the CLI starts the feature as its own compose
|
|
14
|
+
project named after the feature and not after any product. Index names are a
|
|
15
|
+
node-wide namespace, so without a prefix two products indexing "pages" would
|
|
16
|
+
overwrite each other's documents. The feature's .env ships the value
|
|
17
|
+
__PRODUCT__, which product:env --merge and product:build resolve to the
|
|
18
|
+
product name.
|
|
19
|
+
|
|
20
|
+
To regenerate from source code: splent feature:inject-config splent_feature_elasticsearch
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
import os
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _as_bool(value: str) -> bool:
|
|
27
|
+
return value.strip().lower() in ("1", "true", "yes", "on")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def inject_config(app):
|
|
31
|
+
app.config.update(
|
|
32
|
+
{
|
|
33
|
+
"ELASTICSEARCH_URL": os.getenv(
|
|
34
|
+
"ELASTICSEARCH_URL", "http://localhost:9200"
|
|
35
|
+
),
|
|
36
|
+
"ELASTICSEARCH_ENABLED": _as_bool(
|
|
37
|
+
os.getenv("ELASTICSEARCH_ENABLED", "true")
|
|
38
|
+
),
|
|
39
|
+
# Empty on purpose. The service falls back to the name of the
|
|
40
|
+
# product that is running, which is the namespace wanted and is
|
|
41
|
+
# always known inside one. A shared literal here would be a
|
|
42
|
+
# default that every misconfigured product agrees on, which is
|
|
43
|
+
# exactly the collision this variable exists to prevent.
|
|
44
|
+
"ELASTICSEARCH_INDEX_PREFIX": os.getenv("ELASTICSEARCH_INDEX_PREFIX", ""),
|
|
45
|
+
}
|
|
46
|
+
)
|
|
@@ -0,0 +1,469 @@
|
|
|
1
|
+
"""Generic Elasticsearch access for SPLENT products (infrastructure service).
|
|
2
|
+
|
|
3
|
+
A thin, domain-agnostic wrapper around the official ``elasticsearch`` client.
|
|
4
|
+
It owns no relational entity, so it is a plain service rather than a
|
|
5
|
+
repository-backed one. Consumer features decide *what* to index and *how* to
|
|
6
|
+
query; this service only provides the transport:
|
|
7
|
+
|
|
8
|
+
es = service_proxy("ElasticsearchService")
|
|
9
|
+
if es.available():
|
|
10
|
+
es.ensure_index("posts", mappings={...})
|
|
11
|
+
es.index_document("posts", post.id, {"title": post.title})
|
|
12
|
+
|
|
13
|
+
One node, many products
|
|
14
|
+
-----------------------
|
|
15
|
+
In development the CLI starts each feature as its own compose project, named
|
|
16
|
+
after the feature reference with no product name in it, so every product in
|
|
17
|
+
the workspace talks to the same Elasticsearch container and writes into the
|
|
18
|
+
same data volume. Index names are a flat, node-wide namespace, which means
|
|
19
|
+
two products that both index "pages" would overwrite each other's documents
|
|
20
|
+
and answer searches with each other's material. Every index name given to
|
|
21
|
+
this service is therefore prefixed with ELASTICSEARCH_INDEX_PREFIX, which the
|
|
22
|
+
product's .env resolves to the product name, so "pages" becomes
|
|
23
|
+
"innosoft_app_pages" over here and "diversolab_app_pages" over there. Callers
|
|
24
|
+
keep passing their own short names; ``index_name()`` tells them the real one
|
|
25
|
+
when they need to see it.
|
|
26
|
+
|
|
27
|
+
An index is a cache and never an authority on who may read what. This service
|
|
28
|
+
stores and returns whatever documents it is given, and the feature that owns
|
|
29
|
+
the material decides at request time whether a particular reader may see a
|
|
30
|
+
particular hit.
|
|
31
|
+
|
|
32
|
+
Failure modes
|
|
33
|
+
-------------
|
|
34
|
+
Elasticsearch is a soft dependency, so failures are typed rather than raw:
|
|
35
|
+
``available()`` never raises, and every other method raises
|
|
36
|
+
:class:`ElasticsearchError`, either :class:`ElasticsearchUnavailable` when the
|
|
37
|
+
node could not be reached or :class:`ElasticsearchRequestError` when the node
|
|
38
|
+
answered and refused the request. Nothing from the underlying client escapes.
|
|
39
|
+
|
|
40
|
+
Configuration (see config.py): ELASTICSEARCH_URL, ELASTICSEARCH_ENABLED,
|
|
41
|
+
ELASTICSEARCH_INDEX_PREFIX. All are read from app.config when a Flask app
|
|
42
|
+
context is active, falling back to the environment so the service also works
|
|
43
|
+
in plain scripts and tests.
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
import logging
|
|
47
|
+
import os
|
|
48
|
+
from contextlib import contextmanager
|
|
49
|
+
from itertools import batched
|
|
50
|
+
|
|
51
|
+
from flask import current_app, has_app_context
|
|
52
|
+
|
|
53
|
+
logger = logging.getLogger(__name__)
|
|
54
|
+
|
|
55
|
+
# Client-wide request timeout (seconds). Deliberately short so an absent
|
|
56
|
+
# Elasticsearch fails fast instead of hanging requests.
|
|
57
|
+
REQUEST_TIMEOUT = 5
|
|
58
|
+
|
|
59
|
+
DEFAULT_URL = "http://localhost:9200"
|
|
60
|
+
|
|
61
|
+
# Namespace used when ELASTICSEARCH_INDEX_PREFIX is unset, which happens in
|
|
62
|
+
# bare scripts and tests but not in a product: product:env resolves the
|
|
63
|
+
# __PRODUCT__ placeholder shipped in this feature's .env into the product name.
|
|
64
|
+
# Documents per _bulk request. Elastic's own advice is to keep a bulk body in
|
|
65
|
+
# the low megabytes, and a few hundred small documents sit well inside that
|
|
66
|
+
# while turning a ten thousand page reindex into twenty requests.
|
|
67
|
+
DEFAULT_BULK_BATCH_SIZE = 500
|
|
68
|
+
|
|
69
|
+
# How many per-document failures a bulk report carries back. A reindex against
|
|
70
|
+
# a broken mapping fails on every single document, and handing ten thousand
|
|
71
|
+
# error strings to a CLI command helps nobody; the counts stay exact.
|
|
72
|
+
MAX_REPORTED_BULK_FAILURES = 20
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class ElasticsearchError(RuntimeError):
|
|
76
|
+
"""Base class for every failure this service reports.
|
|
77
|
+
|
|
78
|
+
Public methods let nothing from the underlying client escape, so a caller
|
|
79
|
+
that treats Elasticsearch as a soft dependency can catch this one class
|
|
80
|
+
and be sure it has caught everything. Which subclass arrives says whether
|
|
81
|
+
falling back is the right answer or whether there is a bug to fix.
|
|
82
|
+
"""
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
class ElasticsearchUnavailable(ElasticsearchError):
|
|
86
|
+
"""The node could not be reached, or the feature is switched off.
|
|
87
|
+
|
|
88
|
+
Nothing is wrong with the request, so consumers degrade and carry on::
|
|
89
|
+
|
|
90
|
+
try:
|
|
91
|
+
results = es.search("posts", query)
|
|
92
|
+
except ElasticsearchUnavailable:
|
|
93
|
+
results = fallback_search(query)
|
|
94
|
+
"""
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
class ElasticsearchRequestError(ElasticsearchError):
|
|
98
|
+
"""The node answered, refusing the request.
|
|
99
|
+
|
|
100
|
+
A malformed query body, a field the mapping will not accept, a missing
|
|
101
|
+
permission. These used to travel up as raw client exceptions, so a caller
|
|
102
|
+
that had carefully handled the unreachable case still returned a 500 the
|
|
103
|
+
first time somebody mistyped a query. Retrying will not help and a silent
|
|
104
|
+
fallback would hide the defect, so consumers normally let this surface.
|
|
105
|
+
"""
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
# One client per URL, shared across service instances. service_proxy()
|
|
109
|
+
# instantiates a fresh service on every access, so per-instance caching
|
|
110
|
+
# would recreate the client constantly.
|
|
111
|
+
_clients: dict[str, object] = {}
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _build_client(url: str):
|
|
115
|
+
"""Create the raw Elasticsearch client. Isolated so tests can patch it."""
|
|
116
|
+
from elasticsearch import Elasticsearch
|
|
117
|
+
|
|
118
|
+
return Elasticsearch(hosts=[url], request_timeout=REQUEST_TIMEOUT)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def reset_client_cache() -> None:
|
|
122
|
+
"""Drop cached clients (used by tests and after config changes)."""
|
|
123
|
+
_clients.clear()
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _describe_bulk_error(error) -> str:
|
|
127
|
+
"""The one useful line out of a per-item _bulk error object.
|
|
128
|
+
|
|
129
|
+
Elasticsearch reports item failures as a nested object whose ``reason``
|
|
130
|
+
already reads as a sentence, so that is what a log line or a CLI report
|
|
131
|
+
wants; the type is the fallback for the rare error without one.
|
|
132
|
+
"""
|
|
133
|
+
if not isinstance(error, dict):
|
|
134
|
+
return str(error)
|
|
135
|
+
return error.get("reason") or error.get("type") or "unknown error"
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
class ElasticsearchService:
|
|
139
|
+
"""Minimal generic API over an Elasticsearch cluster."""
|
|
140
|
+
|
|
141
|
+
# ── Configuration ────────────────────────────────────────────────────
|
|
142
|
+
|
|
143
|
+
def _cfg(self, key: str, default):
|
|
144
|
+
if has_app_context():
|
|
145
|
+
return current_app.config.get(key, os.getenv(key, default))
|
|
146
|
+
return os.getenv(key, default)
|
|
147
|
+
|
|
148
|
+
@property
|
|
149
|
+
def url(self) -> str:
|
|
150
|
+
return self._cfg("ELASTICSEARCH_URL", DEFAULT_URL)
|
|
151
|
+
|
|
152
|
+
@property
|
|
153
|
+
def enabled(self) -> bool:
|
|
154
|
+
value = self._cfg("ELASTICSEARCH_ENABLED", "true")
|
|
155
|
+
if isinstance(value, str):
|
|
156
|
+
return value.strip().lower() in ("1", "true", "yes", "on")
|
|
157
|
+
return bool(value)
|
|
158
|
+
|
|
159
|
+
@property
|
|
160
|
+
def index_prefix(self) -> str:
|
|
161
|
+
"""The namespace this product's indices live under on the node.
|
|
162
|
+
|
|
163
|
+
Every product on a node shares one flat namespace, and in
|
|
164
|
+
development they really do share a node: the CLI starts each feature
|
|
165
|
+
as its own compose project named after the feature, with no product
|
|
166
|
+
in the name, so one Elasticsearch serves every product in the
|
|
167
|
+
workspace. Two products that landed on the same prefix would write
|
|
168
|
+
into each other's indices, and a reindex by either would delete the
|
|
169
|
+
other's.
|
|
170
|
+
|
|
171
|
+
So an unset prefix, or one still holding the unresolved __PRODUCT__
|
|
172
|
+
placeholder because no product ever merged this feature's .env,
|
|
173
|
+
falls back to the name of the product that is running, which is
|
|
174
|
+
exactly the namespace wanted and is always known inside a product.
|
|
175
|
+
It does NOT fall back to a shared constant: sharing is the failure
|
|
176
|
+
being prevented, and a default that every misconfigured product
|
|
177
|
+
agrees on fails in the direction that hurts.
|
|
178
|
+
"""
|
|
179
|
+
value = str(self._cfg("ELASTICSEARCH_INDEX_PREFIX", "")).strip()
|
|
180
|
+
if not value or "__PRODUCT__" in value:
|
|
181
|
+
value = str(os.getenv("SPLENT_APP", "")).strip()
|
|
182
|
+
if not value:
|
|
183
|
+
raise ElasticsearchRequestError(
|
|
184
|
+
"No index prefix and no SPLENT_APP, so there is no way to tell "
|
|
185
|
+
"this product's indices from another product's on the same "
|
|
186
|
+
"node. Set ELASTICSEARCH_INDEX_PREFIX."
|
|
187
|
+
)
|
|
188
|
+
# Elasticsearch rejects an index name containing an uppercase letter,
|
|
189
|
+
# and product names are free to carry one.
|
|
190
|
+
return value.lower()
|
|
191
|
+
|
|
192
|
+
def index_name(self, name: str) -> str:
|
|
193
|
+
"""The real index name behind a caller's logical one.
|
|
194
|
+
|
|
195
|
+
Callers work in short names ("pages") and this service puts them in
|
|
196
|
+
the product's namespace. Use this when the real name has to be shown,
|
|
197
|
+
logged, or handed to something that talks to the node directly.
|
|
198
|
+
|
|
199
|
+
Prefixing a name that already carries the prefix is a no-op, so a
|
|
200
|
+
caller that reads a name back out of here and passes it to search()
|
|
201
|
+
still queries its own index rather than an empty
|
|
202
|
+
"prefix_prefix_pages" that would quietly return no results.
|
|
203
|
+
"""
|
|
204
|
+
prefix = self.index_prefix
|
|
205
|
+
name = str(name)
|
|
206
|
+
if name == prefix or name.startswith(f"{prefix}_"):
|
|
207
|
+
return name
|
|
208
|
+
return f"{prefix}_{name}"
|
|
209
|
+
|
|
210
|
+
# ── Client ───────────────────────────────────────────────────────────
|
|
211
|
+
|
|
212
|
+
def _client(self):
|
|
213
|
+
"""Lazy, cached client. Raises ElasticsearchUnavailable when disabled."""
|
|
214
|
+
if not self.enabled:
|
|
215
|
+
raise ElasticsearchUnavailable(
|
|
216
|
+
"Elasticsearch is disabled (ELASTICSEARCH_ENABLED=false)."
|
|
217
|
+
)
|
|
218
|
+
url = self.url
|
|
219
|
+
client = _clients.get(url)
|
|
220
|
+
if client is None:
|
|
221
|
+
client = _build_client(url)
|
|
222
|
+
_clients[url] = client
|
|
223
|
+
return client
|
|
224
|
+
|
|
225
|
+
# ── Error translation ────────────────────────────────────────────────
|
|
226
|
+
|
|
227
|
+
@staticmethod
|
|
228
|
+
def _rejection_errors() -> tuple:
|
|
229
|
+
"""Client exceptions meaning the request itself was not acceptable.
|
|
230
|
+
|
|
231
|
+
SerializationError is a TransportError by inheritance, but it is
|
|
232
|
+
raised over a body this caller built rather than over a network that
|
|
233
|
+
went away, so it belongs with the rejections. That is also why this
|
|
234
|
+
tuple has to be tried before the transport one.
|
|
235
|
+
"""
|
|
236
|
+
from elasticsearch import ApiError, SerializationError
|
|
237
|
+
|
|
238
|
+
return (ApiError, SerializationError)
|
|
239
|
+
|
|
240
|
+
@staticmethod
|
|
241
|
+
def _transport_errors() -> tuple:
|
|
242
|
+
"""Client exceptions meaning the request never reached an answer.
|
|
243
|
+
|
|
244
|
+
The two named ones are what a stopped container produces and what
|
|
245
|
+
consumers see day to day; TransportError covers the rest of the family
|
|
246
|
+
(TLS handshake, node sniffing) so nothing raw escapes.
|
|
247
|
+
"""
|
|
248
|
+
from elasticsearch import ConnectionError as ESConnectionError
|
|
249
|
+
from elasticsearch import ConnectionTimeout, TransportError
|
|
250
|
+
|
|
251
|
+
return (ESConnectionError, ConnectionTimeout, TransportError)
|
|
252
|
+
|
|
253
|
+
def _unavailable(self, exc: Exception) -> ElasticsearchUnavailable:
|
|
254
|
+
message = f"Elasticsearch at {self.url} is unreachable: {exc}"
|
|
255
|
+
logger.warning(message)
|
|
256
|
+
return ElasticsearchUnavailable(message)
|
|
257
|
+
|
|
258
|
+
def _rejected(self, exc: Exception) -> ElasticsearchRequestError:
|
|
259
|
+
message = f"Elasticsearch at {self.url} rejected the request: {exc}"
|
|
260
|
+
logger.warning(message)
|
|
261
|
+
return ElasticsearchRequestError(message)
|
|
262
|
+
|
|
263
|
+
@contextmanager
|
|
264
|
+
def _translated_errors(self):
|
|
265
|
+
"""Run a client call and report its failure as one of our two kinds.
|
|
266
|
+
|
|
267
|
+
Every public method wraps its call in this, which is what makes the
|
|
268
|
+
promise in the module docstring true: catching ElasticsearchError
|
|
269
|
+
catches everything this service can do to a caller.
|
|
270
|
+
"""
|
|
271
|
+
try:
|
|
272
|
+
yield
|
|
273
|
+
except self._rejection_errors() as exc:
|
|
274
|
+
raise self._rejected(exc) from exc
|
|
275
|
+
except self._transport_errors() as exc:
|
|
276
|
+
raise self._unavailable(exc) from exc
|
|
277
|
+
|
|
278
|
+
# ── Public API ───────────────────────────────────────────────────────
|
|
279
|
+
|
|
280
|
+
def available(self) -> bool:
|
|
281
|
+
"""True when Elasticsearch is enabled and answers a ping. Never raises."""
|
|
282
|
+
if not self.enabled:
|
|
283
|
+
return False
|
|
284
|
+
try:
|
|
285
|
+
return bool(self._client().ping())
|
|
286
|
+
except Exception as exc:
|
|
287
|
+
logger.debug("Elasticsearch ping failed for %s: %s", self.url, exc)
|
|
288
|
+
return False
|
|
289
|
+
|
|
290
|
+
def ensure_index(self, name: str, mappings: dict | None = None) -> bool:
|
|
291
|
+
"""Create an index if it does not exist. Idempotent.
|
|
292
|
+
|
|
293
|
+
Returns True when the index was created, False when it already existed.
|
|
294
|
+
"""
|
|
295
|
+
from elasticsearch import BadRequestError
|
|
296
|
+
|
|
297
|
+
client = self._client()
|
|
298
|
+
index = self.index_name(name)
|
|
299
|
+
with self._translated_errors():
|
|
300
|
+
try:
|
|
301
|
+
if client.indices.exists(index=index):
|
|
302
|
+
return False
|
|
303
|
+
kwargs = {}
|
|
304
|
+
if mappings:
|
|
305
|
+
kwargs["mappings"] = mappings
|
|
306
|
+
client.indices.create(index=index, **kwargs)
|
|
307
|
+
return True
|
|
308
|
+
except BadRequestError as exc:
|
|
309
|
+
# Two workers racing to create the same index. The loser sees
|
|
310
|
+
# resource_already_exists_exception, still a success for us.
|
|
311
|
+
if "resource_already_exists_exception" in str(exc.message):
|
|
312
|
+
return False
|
|
313
|
+
raise
|
|
314
|
+
|
|
315
|
+
def index_document(self, index: str, id: str, body: dict) -> dict:
|
|
316
|
+
"""Index (create or overwrite) a document. Returns the ES response."""
|
|
317
|
+
client = self._client()
|
|
318
|
+
with self._translated_errors():
|
|
319
|
+
return client.index(index=self.index_name(index), id=id, document=body)
|
|
320
|
+
|
|
321
|
+
def bulk_index(
|
|
322
|
+
self,
|
|
323
|
+
index: str,
|
|
324
|
+
documents,
|
|
325
|
+
batch_size: int = DEFAULT_BULK_BATCH_SIZE,
|
|
326
|
+
) -> dict:
|
|
327
|
+
"""Index many documents in as few requests as possible.
|
|
328
|
+
|
|
329
|
+
``documents`` is any iterable of ``{"id": ..., "body": {...}}`` and is
|
|
330
|
+
consumed lazily, so a full reindex can stream out of a query without
|
|
331
|
+
ever holding the whole corpus in memory. Reindexing a wiki is
|
|
332
|
+
thousands of documents, and one HTTP round trip each turns a
|
|
333
|
+
maintenance command into something nobody runs.
|
|
334
|
+
|
|
335
|
+
Returns ``{"indexed": int, "failed": int, "errors": [...]}``. A
|
|
336
|
+
document the node refuses does not abort the run: ten thousand pages
|
|
337
|
+
that trip over one bad field should still leave 9999 searchable, and
|
|
338
|
+
the caller gets exact counts plus a capped sample of reasons to log.
|
|
339
|
+
Only a failure of the request itself raises.
|
|
340
|
+
"""
|
|
341
|
+
client = self._client()
|
|
342
|
+
real_index = self.index_name(index)
|
|
343
|
+
|
|
344
|
+
indexed = 0
|
|
345
|
+
failed = 0
|
|
346
|
+
errors: list[dict] = []
|
|
347
|
+
|
|
348
|
+
def record_failure(doc_id, reason) -> None:
|
|
349
|
+
nonlocal failed
|
|
350
|
+
failed += 1
|
|
351
|
+
if len(errors) < MAX_REPORTED_BULK_FAILURES:
|
|
352
|
+
errors.append({"id": doc_id, "error": reason})
|
|
353
|
+
|
|
354
|
+
for batch in batched(documents, max(1, int(batch_size))):
|
|
355
|
+
operations: list[dict] = []
|
|
356
|
+
for document in batch:
|
|
357
|
+
doc_id = document.get("id")
|
|
358
|
+
if doc_id is None:
|
|
359
|
+
# Without a stable id the node invents one, so every run
|
|
360
|
+
# would append another copy instead of overwriting.
|
|
361
|
+
record_failure(None, "document has no id")
|
|
362
|
+
continue
|
|
363
|
+
operations.append({"index": {"_id": str(doc_id)}})
|
|
364
|
+
operations.append(document.get("body") or {})
|
|
365
|
+
|
|
366
|
+
if not operations:
|
|
367
|
+
continue
|
|
368
|
+
|
|
369
|
+
with self._translated_errors():
|
|
370
|
+
response = client.bulk(index=real_index, operations=operations)
|
|
371
|
+
|
|
372
|
+
items = response.get("items") or []
|
|
373
|
+
if not response.get("errors"):
|
|
374
|
+
# One item per document, but fall back to the operation count
|
|
375
|
+
# for a node that answered without the detail.
|
|
376
|
+
indexed += len(items) if items else len(operations) // 2
|
|
377
|
+
continue
|
|
378
|
+
|
|
379
|
+
# One key per item, named after the action ("index"), holding the
|
|
380
|
+
# per-document result.
|
|
381
|
+
for item in items:
|
|
382
|
+
result = next(iter(item.values()), {}) if item else {}
|
|
383
|
+
error = result.get("error")
|
|
384
|
+
if error:
|
|
385
|
+
record_failure(result.get("_id"), _describe_bulk_error(error))
|
|
386
|
+
else:
|
|
387
|
+
indexed += 1
|
|
388
|
+
|
|
389
|
+
return {"indexed": indexed, "failed": failed, "errors": errors}
|
|
390
|
+
|
|
391
|
+
def refresh_index(self, name: str) -> bool:
|
|
392
|
+
"""Make just-indexed documents searchable immediately.
|
|
393
|
+
|
|
394
|
+
Elasticsearch is near-real-time (documents become visible after the
|
|
395
|
+
next periodic refresh, ~1s): consumers that index and search within
|
|
396
|
+
the same request call this after their indexing loop.
|
|
397
|
+
|
|
398
|
+
An index that is not there is not an error, the same way deleting a
|
|
399
|
+
missing one is not: asking for documents to become searchable when
|
|
400
|
+
there are none has already happened. It used to raise, which put a
|
|
401
|
+
four hundred and four in front of anyone building an index for the
|
|
402
|
+
first time and made a working reindex read like a broken one.
|
|
403
|
+
Returns whether there was anything to refresh.
|
|
404
|
+
"""
|
|
405
|
+
from elasticsearch import NotFoundError
|
|
406
|
+
|
|
407
|
+
client = self._client()
|
|
408
|
+
with self._translated_errors():
|
|
409
|
+
try:
|
|
410
|
+
client.indices.refresh(index=self.index_name(name))
|
|
411
|
+
return True
|
|
412
|
+
except NotFoundError:
|
|
413
|
+
logger.debug("Index '%s' not found; nothing to refresh.", name)
|
|
414
|
+
return False
|
|
415
|
+
|
|
416
|
+
def delete_document(self, index: str, id: str) -> bool:
|
|
417
|
+
"""Delete a document. Returns False (without failing) if it is missing."""
|
|
418
|
+
from elasticsearch import NotFoundError
|
|
419
|
+
|
|
420
|
+
client = self._client()
|
|
421
|
+
with self._translated_errors():
|
|
422
|
+
try:
|
|
423
|
+
client.delete(index=self.index_name(index), id=id)
|
|
424
|
+
return True
|
|
425
|
+
except NotFoundError:
|
|
426
|
+
logger.debug(
|
|
427
|
+
"Document '%s' not found in '%s'; nothing to delete.", id, index
|
|
428
|
+
)
|
|
429
|
+
return False
|
|
430
|
+
|
|
431
|
+
def search(self, index: str, query_body: dict) -> dict:
|
|
432
|
+
"""Run a raw query body against an index.
|
|
433
|
+
|
|
434
|
+
Returns ``{"hits": [<raw hit dicts>], "total": <int>}`` so consumers
|
|
435
|
+
get both ``_source`` and metadata (``_id``, ``_score``) per hit. What
|
|
436
|
+
comes back is a list of candidates, not a list of things the reader is
|
|
437
|
+
allowed to see; deciding that is the owning feature's job.
|
|
438
|
+
"""
|
|
439
|
+
client = self._client()
|
|
440
|
+
with self._translated_errors():
|
|
441
|
+
response = client.search(index=self.index_name(index), body=query_body)
|
|
442
|
+
|
|
443
|
+
hits = response["hits"]["hits"]
|
|
444
|
+
total = response["hits"]["total"]
|
|
445
|
+
if isinstance(total, dict): # ES >= 7 shape: {"value": n, "relation": "eq"}
|
|
446
|
+
total = total.get("value", len(hits))
|
|
447
|
+
return {"hits": list(hits), "total": total}
|
|
448
|
+
|
|
449
|
+
def delete_index(self, name: str) -> bool:
|
|
450
|
+
"""Delete an index. Returns False (without failing) if it is missing."""
|
|
451
|
+
from elasticsearch import NotFoundError
|
|
452
|
+
|
|
453
|
+
client = self._client()
|
|
454
|
+
with self._translated_errors():
|
|
455
|
+
try:
|
|
456
|
+
client.indices.delete(index=self.index_name(name))
|
|
457
|
+
return True
|
|
458
|
+
except NotFoundError:
|
|
459
|
+
logger.debug("Index '%s' not found; nothing to delete.", name)
|
|
460
|
+
return False
|
|
461
|
+
|
|
462
|
+
def health(self) -> dict:
|
|
463
|
+
"""Status snapshot used by the `feature:elasticsearch status` command."""
|
|
464
|
+
return {
|
|
465
|
+
"url": self.url,
|
|
466
|
+
"enabled": self.enabled,
|
|
467
|
+
"index_prefix": self.index_prefix,
|
|
468
|
+
"available": self.available(),
|
|
469
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Signals for splent_feature_elasticsearch.
|
|
3
|
+
|
|
4
|
+
Use ``define_signal`` to declare signals this feature emits.
|
|
5
|
+
Use ``connect_signal`` to listen to signals from other features.
|
|
6
|
+
|
|
7
|
+
Examples::
|
|
8
|
+
|
|
9
|
+
# Define a signal (this feature emits it):
|
|
10
|
+
from splent_framework.signals.signal_utils import define_signal
|
|
11
|
+
item_created = define_signal("item-created", "splent_feature_elasticsearch")
|
|
12
|
+
|
|
13
|
+
# Connect to a signal from another feature:
|
|
14
|
+
from splent_framework.signals.signal_utils import connect_signal
|
|
15
|
+
|
|
16
|
+
@connect_signal("user-registered", "splent_feature_elasticsearch")
|
|
17
|
+
def on_user_registered(sender, user, **kwargs):
|
|
18
|
+
pass
|
|
19
|
+
"""
|
|
Binary file
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# Spanish translations for PROJECT.
|
|
2
|
+
# Copyright (C) 2026 ORGANIZATION
|
|
3
|
+
# This file is distributed under the same license as the PROJECT project.
|
|
4
|
+
# FIRST AUTHOR <EMAIL@ADDRESS>, 2026.
|
|
5
|
+
#
|
|
6
|
+
msgid ""
|
|
7
|
+
msgstr ""
|
|
8
|
+
"Project-Id-Version: PROJECT VERSION\n"
|
|
9
|
+
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
|
|
10
|
+
"POT-Creation-Date: 2026-07-30 15:44+0000\n"
|
|
11
|
+
"PO-Revision-Date: 2026-07-30 15:45+0000\n"
|
|
12
|
+
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
|
|
13
|
+
"Language: es\n"
|
|
14
|
+
"Language-Team: es <LL@li.org>\n"
|
|
15
|
+
"Plural-Forms: nplurals=2; plural=(n != 1);\n"
|
|
16
|
+
"MIME-Version: 1.0\n"
|
|
17
|
+
"Content-Type: text/plain; charset=utf-8\n"
|
|
18
|
+
"Content-Transfer-Encoding: 8bit\n"
|
|
19
|
+
"Generated-By: Babel 2.18.0\n"
|
|
20
|
+
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# Translations template for PROJECT.
|
|
2
|
+
# Copyright (C) 2026 ORGANIZATION
|
|
3
|
+
# This file is distributed under the same license as the PROJECT project.
|
|
4
|
+
# FIRST AUTHOR <EMAIL@ADDRESS>, 2026.
|
|
5
|
+
#
|
|
6
|
+
#, fuzzy
|
|
7
|
+
msgid ""
|
|
8
|
+
msgstr ""
|
|
9
|
+
"Project-Id-Version: PROJECT VERSION\n"
|
|
10
|
+
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
|
|
11
|
+
"POT-Creation-Date: 2026-07-30 15:44+0000\n"
|
|
12
|
+
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
|
|
13
|
+
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
|
|
14
|
+
"Language-Team: LANGUAGE <LL@li.org>\n"
|
|
15
|
+
"MIME-Version: 1.0\n"
|
|
16
|
+
"Content-Type: text/plain; charset=utf-8\n"
|
|
17
|
+
"Content-Transfer-Encoding: 8bit\n"
|
|
18
|
+
"Generated-By: Babel 2.18.0\n"
|
|
19
|
+
|