ifdd-ts-core 26.10.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ifdd_ts_core-26.10.2/LICENSE +28 -0
- ifdd_ts_core-26.10.2/PKG-INFO +250 -0
- ifdd_ts_core-26.10.2/README.md +226 -0
- ifdd_ts_core-26.10.2/pyproject.toml +47 -0
- ifdd_ts_core-26.10.2/setup.cfg +4 -0
- ifdd_ts_core-26.10.2/src/ifdd/ts/core/__init__.py +13 -0
- ifdd_ts_core-26.10.2/src/ifdd/ts/core/app.py +96 -0
- ifdd_ts_core-26.10.2/src/ifdd/ts/core/auth.py +171 -0
- ifdd_ts_core-26.10.2/src/ifdd/ts/core/config.py +146 -0
- ifdd_ts_core-26.10.2/src/ifdd/ts/core/docs_assets/assets.json +34 -0
- ifdd_ts_core-26.10.2/src/ifdd/ts/core/docs_assets/ifdd/docs.css +6 -0
- ifdd_ts_core-26.10.2/src/ifdd/ts/core/docs_assets/ifdd/swagger-init.js +35 -0
- ifdd_ts_core-26.10.2/src/ifdd/ts/core/docs_assets/redoc/LICENSE +22 -0
- ifdd_ts_core-26.10.2/src/ifdd/ts/core/docs_assets/redoc/redoc.standalone.js +1851 -0
- ifdd_ts_core-26.10.2/src/ifdd/ts/core/docs_assets/redoc/redoc.standalone.js.LICENSE.txt +102 -0
- ifdd_ts_core-26.10.2/src/ifdd/ts/core/docs_assets/swagger-ui/LICENSE +202 -0
- ifdd_ts_core-26.10.2/src/ifdd/ts/core/docs_assets/swagger-ui/NOTICE +2 -0
- ifdd_ts_core-26.10.2/src/ifdd/ts/core/docs_assets/swagger-ui/oauth2-redirect.js +1 -0
- ifdd_ts_core-26.10.2/src/ifdd/ts/core/docs_assets/swagger-ui/swagger-ui-bundle.js +2 -0
- ifdd_ts_core-26.10.2/src/ifdd/ts/core/docs_assets/swagger-ui/swagger-ui-bundle.js.LICENSE.txt +129 -0
- ifdd_ts_core-26.10.2/src/ifdd/ts/core/docs_assets/swagger-ui/swagger-ui.css +3 -0
- ifdd_ts_core-26.10.2/src/ifdd/ts/core/docs_ui.py +113 -0
- ifdd_ts_core-26.10.2/src/ifdd/ts/core/errors.py +73 -0
- ifdd_ts_core-26.10.2/src/ifdd/ts/core/http.py +37 -0
- ifdd_ts_core-26.10.2/src/ifdd/ts/core/logging.py +22 -0
- ifdd_ts_core-26.10.2/src/ifdd/ts/core/openapi.py +143 -0
- ifdd_ts_core-26.10.2/src/ifdd/ts/core/routers/__init__.py +6 -0
- ifdd_ts_core-26.10.2/src/ifdd/ts/core/routers/meta.py +56 -0
- ifdd_ts_core-26.10.2/src/ifdd/ts/core/routers/samples.py +315 -0
- ifdd_ts_core-26.10.2/src/ifdd/ts/core/schemas.py +174 -0
- ifdd_ts_core-26.10.2/src/ifdd/ts/core/stores.py +103 -0
- ifdd_ts_core-26.10.2/src/ifdd/ts/core/times.py +57 -0
- ifdd_ts_core-26.10.2/src/ifdd/ts/core/version.py +29 -0
- ifdd_ts_core-26.10.2/src/ifdd/ts/core/vm.py +200 -0
- ifdd_ts_core-26.10.2/src/ifdd/ts/core/vocabulary.py +196 -0
- ifdd_ts_core-26.10.2/src/ifdd_ts_core.egg-info/PKG-INFO +250 -0
- ifdd_ts_core-26.10.2/src/ifdd_ts_core.egg-info/SOURCES.txt +46 -0
- ifdd_ts_core-26.10.2/src/ifdd_ts_core.egg-info/dependency_links.txt +1 -0
- ifdd_ts_core-26.10.2/src/ifdd_ts_core.egg-info/requires.txt +14 -0
- ifdd_ts_core-26.10.2/src/ifdd_ts_core.egg-info/top_level.txt +1 -0
- ifdd_ts_core-26.10.2/tests/test_api.py +281 -0
- ifdd_ts_core-26.10.2/tests/test_app.py +73 -0
- ifdd_ts_core-26.10.2/tests/test_auth.py +159 -0
- ifdd_ts_core-26.10.2/tests/test_docs_assets.py +49 -0
- ifdd_ts_core-26.10.2/tests/test_stores.py +65 -0
- ifdd_ts_core-26.10.2/tests/test_times.py +47 -0
- ifdd_ts_core-26.10.2/tests/test_vm_integration.py +135 -0
- ifdd_ts_core-26.10.2/tests/test_vocabulary.py +106 -0
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
BSD 3-Clause License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026, Markus Leist (markus@lei.st)
|
|
4
|
+
|
|
5
|
+
Redistribution and use in source and binary forms, with or without
|
|
6
|
+
modification, are permitted provided that the following conditions are met:
|
|
7
|
+
|
|
8
|
+
1. Redistributions of source code must retain the above copyright notice, this
|
|
9
|
+
list of conditions and the following disclaimer.
|
|
10
|
+
|
|
11
|
+
2. Redistributions in binary form must reproduce the above copyright notice,
|
|
12
|
+
this list of conditions and the following disclaimer in the documentation
|
|
13
|
+
and/or other materials provided with the distribution.
|
|
14
|
+
|
|
15
|
+
3. Neither the name of the copyright holder nor the names of its
|
|
16
|
+
contributors may be used to endorse or promote products derived from
|
|
17
|
+
this software without specific prior written permission.
|
|
18
|
+
|
|
19
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
20
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
21
|
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
22
|
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
|
23
|
+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
24
|
+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
25
|
+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
26
|
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
27
|
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
28
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: ifdd-ts-core
|
|
3
|
+
Version: 26.10.2
|
|
4
|
+
Summary: Shared runtime of the IFDD time series API (ifdd-ts-api)
|
|
5
|
+
License-Expression: BSD-3-Clause
|
|
6
|
+
Project-URL: Repository, https://github.com/ifdd-de/ts-core
|
|
7
|
+
Requires-Python: >=3.11
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Requires-Dist: fastapi~=0.142.2
|
|
11
|
+
Requires-Dist: uvicorn[standard]~=0.54.0
|
|
12
|
+
Requires-Dist: pydantic~=2.13.5
|
|
13
|
+
Requires-Dist: pydantic-settings~=2.15.0
|
|
14
|
+
Requires-Dist: httpx~=0.28.1
|
|
15
|
+
Requires-Dist: pyld~=3.3.0
|
|
16
|
+
Requires-Dist: cachetools~=7.2.1
|
|
17
|
+
Requires-Dist: PyJWT[crypto]~=2.15.1
|
|
18
|
+
Provides-Extra: dev
|
|
19
|
+
Requires-Dist: pytest~=9.1.1; extra == "dev"
|
|
20
|
+
Requires-Dist: pytest-asyncio~=1.4.0; extra == "dev"
|
|
21
|
+
Requires-Dist: pytest-cov; extra == "dev"
|
|
22
|
+
Requires-Dist: httpx2; extra == "dev"
|
|
23
|
+
Dynamic: license-file
|
|
24
|
+
|
|
25
|
+
# ifdd-ts-core
|
|
26
|
+
|
|
27
|
+
Shared runtime of the IFDD time series API. Instances are generated from the
|
|
28
|
+
template [`ifdd-de/ts-api`](https://github.com/ifdd-de/ts-api); this package
|
|
29
|
+
holds everything they have in common: token and store checks, the check of
|
|
30
|
+
samples against the vocabularies of the type service, reading and writing in
|
|
31
|
+
VictoriaMetrics, and the documentation pages.
|
|
32
|
+
|
|
33
|
+
```text
|
|
34
|
+
service (client credentials) --HTTPS--> proxy --> ts-api (ifdd-ts-core) --> VictoriaMetrics
|
|
35
|
+
| ^
|
|
36
|
+
+--> type service |
|
|
37
|
+
Grafana (ifdd-ts) reads all ----+
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
- Distribution `ifdd-ts-core`, import `ifdd.ts.core` (`ifdd` and `ifdd.ts`
|
|
41
|
+
are namespace packages). Python 3.11 or later.
|
|
42
|
+
- License BSD-3-Clause. Swagger UI and ReDoc are included as third-party files
|
|
43
|
+
(see below).
|
|
44
|
+
|
|
45
|
+
## Use
|
|
46
|
+
|
|
47
|
+
```python
|
|
48
|
+
from ifdd.ts.core import create_app
|
|
49
|
+
from app.settings import settings # a subclass of ifdd.ts.core.CoreSettings
|
|
50
|
+
|
|
51
|
+
app = create_app(settings)
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
or `uvicorn --factory ifdd.ts.core:create_app` with the settings in the
|
|
55
|
+
environment. Every endpoint is below the path prefix `IFDD_TS_API_PATH`
|
|
56
|
+
(default `/tsapi`), so the API can share a host with Grafana behind the
|
|
57
|
+
proxy without a rewrite rule.
|
|
58
|
+
|
|
59
|
+
## Configuration
|
|
60
|
+
|
|
61
|
+
Environment variables (or `.env`):
|
|
62
|
+
|
|
63
|
+
| Variable | Default | Meaning |
|
|
64
|
+
|---|---|---|
|
|
65
|
+
| `IFDD_TS_API_PATH` | `/tsapi` | path prefix of every endpoint; empty serves at the root |
|
|
66
|
+
| `IFDD_TS_API_VERSION` | package version | version shown by `/version` and the OpenAPI document |
|
|
67
|
+
| `IFDD_TS_API_AUTH_ENABLED` | `true` | `false`: no token is checked, every caller is a development user with both roles and the pattern `IFDD_CONTEXT_ROOT` |
|
|
68
|
+
| `IFDD_OIDC_ISSUER` | – | expected `iss`; empty: not checked (warning) |
|
|
69
|
+
| `IFDD_OIDC_JWKS_URL` | – | keys of the realm |
|
|
70
|
+
| `IFDD_OIDC_TOKEN_URL` | – | only shown in the API description |
|
|
71
|
+
| `IFDD_OIDC_JWKS_AUDIENCE` | `ifdd-ts-api` | required `aud`; the client that carries the roles |
|
|
72
|
+
| `IFDD_TS_ROLE_READ`, `IFDD_TS_ROLE_WRITE` | `ifdd_ts_ro`, `ifdd_ts_rw` | roles to read and to write |
|
|
73
|
+
| `IFDD_CONTEXT_ROOT` | `ifdd.de` | store pattern of the development user |
|
|
74
|
+
| `IFDD_TS_VM_URL` | `http://victoriametrics:8428` | VictoriaMetrics |
|
|
75
|
+
| `IFDD_TYPE_URL` | `http://type/v1` | type service; every property must be below it |
|
|
76
|
+
| `IFDD_TS_MAX_ITEMS` | `10000` | most samples written or read, and most series listed, per request |
|
|
77
|
+
| `IFDD_TS_DEFAULT_RANGE_DAYS` | `30` | time range of `/stores` and `/series` without `start` |
|
|
78
|
+
| `IFDD_TS_LOOKBACK_DAYS` | `3660` | how far back the latest sample is searched without `max_age` |
|
|
79
|
+
| `IFDD_TS_VOCABULARY_CACHE_TTL` | `300` | seconds a vocabulary is kept |
|
|
80
|
+
| `IFDD_TOKEN_CACHE_SIZE`, `IFDD_TOKEN_CACHE_TTL` | `1000`, `300` | cache of validated tokens |
|
|
81
|
+
| `IFDD_TS_VM_TIMEOUT`, `IFDD_TS_VOCABULARY_TIMEOUT` | `30`, `10` | seconds |
|
|
82
|
+
|
|
83
|
+
The limit of 10 000 is a starting value, not a measured one.
|
|
84
|
+
|
|
85
|
+
TLS: calls to the type service (and to the JWKS URL) check certificates
|
|
86
|
+
against the system trust store, not against `certifi`; put the CA of the
|
|
87
|
+
proxy there, as the ts-api image does.
|
|
88
|
+
|
|
89
|
+
## Endpoints
|
|
90
|
+
|
|
91
|
+
All below `IFDD_TS_API_PATH`:
|
|
92
|
+
|
|
93
|
+
| Method and path | Role | Purpose |
|
|
94
|
+
|---|---|---|
|
|
95
|
+
| `GET /version` | none | version (health checks) |
|
|
96
|
+
| `GET /whoami` | valid token | roles and store patterns read from the token |
|
|
97
|
+
| `GET /stores` | read | stores with samples that the caller may read |
|
|
98
|
+
| `GET /stores/{store}/series` | read + pattern | series of a store, filtered by labels |
|
|
99
|
+
| `GET /stores/{store}/samples` | read + pattern | latest sample of one series at or before `at` |
|
|
100
|
+
| `GET /stores/{store}/samples/range` | read + pattern | raw samples of the series of an object in `[start, end]` |
|
|
101
|
+
| `POST /stores/{store}/samples` | write + pattern | write one sample |
|
|
102
|
+
| `POST /stores/{store}/samples/batch` | write + pattern | write a list of samples (all or nothing on checks) |
|
|
103
|
+
| `GET /docs`, `GET /redoc`, `GET /openapi.json` | none | documentation |
|
|
104
|
+
|
|
105
|
+
Order of checks: token (401), store name (422), role (403), store pattern
|
|
106
|
+
(403), body (422), vocabulary (422, or 502 if the type service fails),
|
|
107
|
+
VictoriaMetrics (502).
|
|
108
|
+
|
|
109
|
+
## Stores and access
|
|
110
|
+
|
|
111
|
+
A store is a named part of the one VictoriaMetrics instance: every sample
|
|
112
|
+
carries the label `ifdd_store`. The API passes `extra_label=ifdd_store=<store>`
|
|
113
|
+
on every call that reads or writes samples. VictoriaMetrics then sets the
|
|
114
|
+
label on every imported sample, overriding one in the body, and adds it as a
|
|
115
|
+
filter to every read (checked against v1.153.0). That parameter alone keeps
|
|
116
|
+
stores apart; the tests fail without it.
|
|
117
|
+
|
|
118
|
+
Store names are written like DNS names: at least two labels of lower-case
|
|
119
|
+
letters, digits and inner hyphens, e.g. `temperature.ops.example.ifdd.de`.
|
|
120
|
+
Any such name is a store; it exists once a sample was written to it.
|
|
121
|
+
|
|
122
|
+
Callers are services with an access token of the client credentials grant.
|
|
123
|
+
The token's audience must be `IFDD_OIDC_JWKS_AUDIENCE`, and its client roles
|
|
124
|
+
there decide:
|
|
125
|
+
|
|
126
|
+
- `ifdd_ts_ro` reads, `ifdd_ts_rw` writes; they apply to every store the
|
|
127
|
+
caller's patterns open;
|
|
128
|
+
- a role with a dot is a **store pattern**. It opens the store of that name
|
|
129
|
+
and every store below it, compared at label boundaries
|
|
130
|
+
(`ops.example.ifdd.de` opens `temperature.ops.example.ifdd.de`, not
|
|
131
|
+
`xops.example.ifdd.de`). The root `ifdd.de` opens all.
|
|
132
|
+
|
|
133
|
+
A foreign store answers 403, not 404. `/stores` leaves out stores the caller
|
|
134
|
+
may not read.
|
|
135
|
+
|
|
136
|
+
**What this does not protect.** Grafana (ifdd-ts) reads VictoriaMetrics
|
|
137
|
+
directly and shows the series of every store; containers on the ingest
|
|
138
|
+
network write to VictoriaMetrics directly, without any check. Both are
|
|
139
|
+
deliberate (see the documentation of ifdd-ts); the API is the way in for
|
|
140
|
+
services that should not get either.
|
|
141
|
+
|
|
142
|
+
## Series and samples
|
|
143
|
+
|
|
144
|
+
A series is named by four labels, the series contract of ifdd-ts:
|
|
145
|
+
|
|
146
|
+
| Field | Label | Value |
|
|
147
|
+
|---|---|---|
|
|
148
|
+
| `name` | `__name__` | metric name, only for people: letters, digits, `_` |
|
|
149
|
+
| `ifdd_id` | `ifdd_id` | full IRI of the object in ifdd-be |
|
|
150
|
+
| `ifdd_property` | `ifdd_property` | full IRI of the property in the type service |
|
|
151
|
+
| `ifdd_unit` | `ifdd_unit` | full IRI of the unit |
|
|
152
|
+
|
|
153
|
+
No further labels are accepted. A value is a finite number; states are coded
|
|
154
|
+
as numbers.
|
|
155
|
+
|
|
156
|
+
**Times** are ISO 8601 with an offset (`Z`, `+02:00`), at most to the
|
|
157
|
+
millisecond, not before 1970. Without offset or with finer precision the API
|
|
158
|
+
answers 422 instead of guessing or cutting. Answers are UTC with three
|
|
159
|
+
decimals (`2026-10-22T07:15:00.250+00:00`).
|
|
160
|
+
|
|
161
|
+
**Appending only.** There is no PUT: VictoriaMetrics cannot replace a single
|
|
162
|
+
sample. Two samples of the same series and time are both kept, unless
|
|
163
|
+
deduplication is on (`-dedup.minScrapeInterval=1ms`, as in ifdd-ts); then
|
|
164
|
+
the **larger** value stays, not the later one (checked against v1.153.0).
|
|
165
|
+
Sending the same sample twice is therefore harmless.
|
|
166
|
+
|
|
167
|
+
**Latest sample.** `GET /stores/{store}/samples` needs `ifdd_id` and
|
|
168
|
+
`ifdd_property`; if more than one series has them (other name or unit), it
|
|
169
|
+
answers 422 and asks for `name` and `ifdd_unit`. It searches back from `at`
|
|
170
|
+
in growing windows (one hour, one day, 30 days, 366 days, then
|
|
171
|
+
`IFDD_TS_LOOKBACK_DAYS` or `max_age`), so a recent sample costs one small
|
|
172
|
+
read.
|
|
173
|
+
|
|
174
|
+
## Checks against the vocabulary
|
|
175
|
+
|
|
176
|
+
Before writing, every sample is checked against the vocabulary of its
|
|
177
|
+
property in the type service:
|
|
178
|
+
|
|
179
|
+
1. `ifdd_property` must be below `IFDD_TYPE_URL`. Its namespace is the IRI up
|
|
180
|
+
to the last `/` or `#`; the type service answers the namespace IRI with
|
|
181
|
+
the vocabulary.
|
|
182
|
+
2. The vocabulary must define the property as an `rdf:Property`.
|
|
183
|
+
3. If the property lists `qudt:applicableUnit`, `ifdd_unit` must be one of
|
|
184
|
+
them.
|
|
185
|
+
4. If it has `sh:minInclusive` or `sh:maxInclusive`, the value must lie
|
|
186
|
+
within.
|
|
187
|
+
|
|
188
|
+
The vocabulary is read with a JSON-LD processor, so prefixes and term names
|
|
189
|
+
do not matter; its context must be inline. A vocabulary is kept for
|
|
190
|
+
`IFDD_TS_VOCABULARY_CACHE_TTL` seconds; failures are not kept. A batch is
|
|
191
|
+
checked completely before anything is written; the 422 answer names every
|
|
192
|
+
bad sample by its index.
|
|
193
|
+
|
|
194
|
+
## Relation to ifdd-be-core
|
|
195
|
+
|
|
196
|
+
The token check and the documentation pages are copies of the ones in
|
|
197
|
+
`ifdd-be-core`, by decision (a shared package would have made this one
|
|
198
|
+
depend on PostgreSQL, rdflib and the backend settings). Fixes in one belong
|
|
199
|
+
in the other. If a third service needs them, they move into a package of
|
|
200
|
+
their own. One difference on purpose: a cached token is checked for expiry
|
|
201
|
+
on every use.
|
|
202
|
+
|
|
203
|
+
## Development
|
|
204
|
+
|
|
205
|
+
```bash
|
|
206
|
+
pip install -e '.[dev]'
|
|
207
|
+
pytest
|
|
208
|
+
```
|
|
209
|
+
|
|
210
|
+
`tests/test_vm_integration.py` runs against a real VictoriaMetrics when
|
|
211
|
+
`IFDD_TEST_VM_URL` is set; start it with deduplication like ifdd-ts:
|
|
212
|
+
|
|
213
|
+
```bash
|
|
214
|
+
victoria-metrics-prod -storageDataPath=/tmp/vm -httpListenAddr=127.0.0.1:8428 \
|
|
215
|
+
-dedup.minScrapeInterval=1ms -retentionPeriod=10y &
|
|
216
|
+
IFDD_TEST_VM_URL=http://127.0.0.1:8428 pytest
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
The CI does the same with the release binary v1.153.0 (SHA-256 checked).
|
|
220
|
+
|
|
221
|
+
Documentation: `docs/en` and `docs/de` (Sphinx, MyST);
|
|
222
|
+
`python tests/check_docs.py . --prefix ifdd-ts-core-` after exporting the
|
|
223
|
+
OpenAPI document to `docs/<lang>/openapi.json` (see `api-http.md`).
|
|
224
|
+
|
|
225
|
+
## Third-party files
|
|
226
|
+
|
|
227
|
+
`src/ifdd/ts/core/docs_assets/` holds unchanged copies of these npm packages,
|
|
228
|
+
the same as in ifdd-be-core; `docs_assets/assets.json` names version,
|
|
229
|
+
tarball, npm integrity and the SHA-256 of every file, and
|
|
230
|
+
`tests/test_docs_assets.py` checks them.
|
|
231
|
+
|
|
232
|
+
| Package | Version | License |
|
|
233
|
+
|---|---|---|
|
|
234
|
+
| `swagger-ui-dist` | 5.33.0 | Apache-2.0 (with `NOTICE`) |
|
|
235
|
+
| `redoc` | 2.5.4 | MIT |
|
|
236
|
+
|
|
237
|
+
The license texts are shipped next to the files and in the wheel; the bundles
|
|
238
|
+
contain code under further licenses (see the `*.LICENSE.txt` files).
|
|
239
|
+
`docs_assets/ifdd/` is part of this package (BSD-3-Clause). Update them
|
|
240
|
+
together with ifdd-be-core, as described in its README.
|
|
241
|
+
|
|
242
|
+
## AI assistance
|
|
243
|
+
|
|
244
|
+
Parts of this repository, especially tests, test data and documentation,
|
|
245
|
+
were developed or improved with the help of large language models. The
|
|
246
|
+
maintainer reviewed and approved every change.
|
|
247
|
+
|
|
248
|
+
## License
|
|
249
|
+
|
|
250
|
+
BSD-3-Clause, see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
# ifdd-ts-core
|
|
2
|
+
|
|
3
|
+
Shared runtime of the IFDD time series API. Instances are generated from the
|
|
4
|
+
template [`ifdd-de/ts-api`](https://github.com/ifdd-de/ts-api); this package
|
|
5
|
+
holds everything they have in common: token and store checks, the check of
|
|
6
|
+
samples against the vocabularies of the type service, reading and writing in
|
|
7
|
+
VictoriaMetrics, and the documentation pages.
|
|
8
|
+
|
|
9
|
+
```text
|
|
10
|
+
service (client credentials) --HTTPS--> proxy --> ts-api (ifdd-ts-core) --> VictoriaMetrics
|
|
11
|
+
| ^
|
|
12
|
+
+--> type service |
|
|
13
|
+
Grafana (ifdd-ts) reads all ----+
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
- Distribution `ifdd-ts-core`, import `ifdd.ts.core` (`ifdd` and `ifdd.ts`
|
|
17
|
+
are namespace packages). Python 3.11 or later.
|
|
18
|
+
- License BSD-3-Clause. Swagger UI and ReDoc are included as third-party files
|
|
19
|
+
(see below).
|
|
20
|
+
|
|
21
|
+
## Use
|
|
22
|
+
|
|
23
|
+
```python
|
|
24
|
+
from ifdd.ts.core import create_app
|
|
25
|
+
from app.settings import settings # a subclass of ifdd.ts.core.CoreSettings
|
|
26
|
+
|
|
27
|
+
app = create_app(settings)
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
or `uvicorn --factory ifdd.ts.core:create_app` with the settings in the
|
|
31
|
+
environment. Every endpoint is below the path prefix `IFDD_TS_API_PATH`
|
|
32
|
+
(default `/tsapi`), so the API can share a host with Grafana behind the
|
|
33
|
+
proxy without a rewrite rule.
|
|
34
|
+
|
|
35
|
+
## Configuration
|
|
36
|
+
|
|
37
|
+
Environment variables (or `.env`):
|
|
38
|
+
|
|
39
|
+
| Variable | Default | Meaning |
|
|
40
|
+
|---|---|---|
|
|
41
|
+
| `IFDD_TS_API_PATH` | `/tsapi` | path prefix of every endpoint; empty serves at the root |
|
|
42
|
+
| `IFDD_TS_API_VERSION` | package version | version shown by `/version` and the OpenAPI document |
|
|
43
|
+
| `IFDD_TS_API_AUTH_ENABLED` | `true` | `false`: no token is checked, every caller is a development user with both roles and the pattern `IFDD_CONTEXT_ROOT` |
|
|
44
|
+
| `IFDD_OIDC_ISSUER` | – | expected `iss`; empty: not checked (warning) |
|
|
45
|
+
| `IFDD_OIDC_JWKS_URL` | – | keys of the realm |
|
|
46
|
+
| `IFDD_OIDC_TOKEN_URL` | – | only shown in the API description |
|
|
47
|
+
| `IFDD_OIDC_JWKS_AUDIENCE` | `ifdd-ts-api` | required `aud`; the client that carries the roles |
|
|
48
|
+
| `IFDD_TS_ROLE_READ`, `IFDD_TS_ROLE_WRITE` | `ifdd_ts_ro`, `ifdd_ts_rw` | roles to read and to write |
|
|
49
|
+
| `IFDD_CONTEXT_ROOT` | `ifdd.de` | store pattern of the development user |
|
|
50
|
+
| `IFDD_TS_VM_URL` | `http://victoriametrics:8428` | VictoriaMetrics |
|
|
51
|
+
| `IFDD_TYPE_URL` | `http://type/v1` | type service; every property must be below it |
|
|
52
|
+
| `IFDD_TS_MAX_ITEMS` | `10000` | most samples written or read, and most series listed, per request |
|
|
53
|
+
| `IFDD_TS_DEFAULT_RANGE_DAYS` | `30` | time range of `/stores` and `/series` without `start` |
|
|
54
|
+
| `IFDD_TS_LOOKBACK_DAYS` | `3660` | how far back the latest sample is searched without `max_age` |
|
|
55
|
+
| `IFDD_TS_VOCABULARY_CACHE_TTL` | `300` | seconds a vocabulary is kept |
|
|
56
|
+
| `IFDD_TOKEN_CACHE_SIZE`, `IFDD_TOKEN_CACHE_TTL` | `1000`, `300` | cache of validated tokens |
|
|
57
|
+
| `IFDD_TS_VM_TIMEOUT`, `IFDD_TS_VOCABULARY_TIMEOUT` | `30`, `10` | seconds |
|
|
58
|
+
|
|
59
|
+
The limit of 10 000 is a starting value, not a measured one.
|
|
60
|
+
|
|
61
|
+
TLS: calls to the type service (and to the JWKS URL) check certificates
|
|
62
|
+
against the system trust store, not against `certifi`; put the CA of the
|
|
63
|
+
proxy there, as the ts-api image does.
|
|
64
|
+
|
|
65
|
+
## Endpoints
|
|
66
|
+
|
|
67
|
+
All below `IFDD_TS_API_PATH`:
|
|
68
|
+
|
|
69
|
+
| Method and path | Role | Purpose |
|
|
70
|
+
|---|---|---|
|
|
71
|
+
| `GET /version` | none | version (health checks) |
|
|
72
|
+
| `GET /whoami` | valid token | roles and store patterns read from the token |
|
|
73
|
+
| `GET /stores` | read | stores with samples that the caller may read |
|
|
74
|
+
| `GET /stores/{store}/series` | read + pattern | series of a store, filtered by labels |
|
|
75
|
+
| `GET /stores/{store}/samples` | read + pattern | latest sample of one series at or before `at` |
|
|
76
|
+
| `GET /stores/{store}/samples/range` | read + pattern | raw samples of the series of an object in `[start, end]` |
|
|
77
|
+
| `POST /stores/{store}/samples` | write + pattern | write one sample |
|
|
78
|
+
| `POST /stores/{store}/samples/batch` | write + pattern | write a list of samples (all or nothing on checks) |
|
|
79
|
+
| `GET /docs`, `GET /redoc`, `GET /openapi.json` | none | documentation |
|
|
80
|
+
|
|
81
|
+
Order of checks: token (401), store name (422), role (403), store pattern
|
|
82
|
+
(403), body (422), vocabulary (422, or 502 if the type service fails),
|
|
83
|
+
VictoriaMetrics (502).
|
|
84
|
+
|
|
85
|
+
## Stores and access
|
|
86
|
+
|
|
87
|
+
A store is a named part of the one VictoriaMetrics instance: every sample
|
|
88
|
+
carries the label `ifdd_store`. The API passes `extra_label=ifdd_store=<store>`
|
|
89
|
+
on every call that reads or writes samples. VictoriaMetrics then sets the
|
|
90
|
+
label on every imported sample, overriding one in the body, and adds it as a
|
|
91
|
+
filter to every read (checked against v1.153.0). That parameter alone keeps
|
|
92
|
+
stores apart; the tests fail without it.
|
|
93
|
+
|
|
94
|
+
Store names are written like DNS names: at least two labels of lower-case
|
|
95
|
+
letters, digits and inner hyphens, e.g. `temperature.ops.example.ifdd.de`.
|
|
96
|
+
Any such name is a store; it exists once a sample was written to it.
|
|
97
|
+
|
|
98
|
+
Callers are services with an access token of the client credentials grant.
|
|
99
|
+
The token's audience must be `IFDD_OIDC_JWKS_AUDIENCE`, and its client roles
|
|
100
|
+
there decide:
|
|
101
|
+
|
|
102
|
+
- `ifdd_ts_ro` reads, `ifdd_ts_rw` writes; they apply to every store the
|
|
103
|
+
caller's patterns open;
|
|
104
|
+
- a role with a dot is a **store pattern**. It opens the store of that name
|
|
105
|
+
and every store below it, compared at label boundaries
|
|
106
|
+
(`ops.example.ifdd.de` opens `temperature.ops.example.ifdd.de`, not
|
|
107
|
+
`xops.example.ifdd.de`). The root `ifdd.de` opens all.
|
|
108
|
+
|
|
109
|
+
A foreign store answers 403, not 404. `/stores` leaves out stores the caller
|
|
110
|
+
may not read.
|
|
111
|
+
|
|
112
|
+
**What this does not protect.** Grafana (ifdd-ts) reads VictoriaMetrics
|
|
113
|
+
directly and shows the series of every store; containers on the ingest
|
|
114
|
+
network write to VictoriaMetrics directly, without any check. Both are
|
|
115
|
+
deliberate (see the documentation of ifdd-ts); the API is the way in for
|
|
116
|
+
services that should not get either.
|
|
117
|
+
|
|
118
|
+
## Series and samples
|
|
119
|
+
|
|
120
|
+
A series is named by four labels, the series contract of ifdd-ts:
|
|
121
|
+
|
|
122
|
+
| Field | Label | Value |
|
|
123
|
+
|---|---|---|
|
|
124
|
+
| `name` | `__name__` | metric name, only for people: letters, digits, `_` |
|
|
125
|
+
| `ifdd_id` | `ifdd_id` | full IRI of the object in ifdd-be |
|
|
126
|
+
| `ifdd_property` | `ifdd_property` | full IRI of the property in the type service |
|
|
127
|
+
| `ifdd_unit` | `ifdd_unit` | full IRI of the unit |
|
|
128
|
+
|
|
129
|
+
No further labels are accepted. A value is a finite number; states are coded
|
|
130
|
+
as numbers.
|
|
131
|
+
|
|
132
|
+
**Times** are ISO 8601 with an offset (`Z`, `+02:00`), at most to the
|
|
133
|
+
millisecond, not before 1970. Without offset or with finer precision the API
|
|
134
|
+
answers 422 instead of guessing or cutting. Answers are UTC with three
|
|
135
|
+
decimals (`2026-10-22T07:15:00.250+00:00`).
|
|
136
|
+
|
|
137
|
+
**Appending only.** There is no PUT: VictoriaMetrics cannot replace a single
|
|
138
|
+
sample. Two samples of the same series and time are both kept, unless
|
|
139
|
+
deduplication is on (`-dedup.minScrapeInterval=1ms`, as in ifdd-ts); then
|
|
140
|
+
the **larger** value stays, not the later one (checked against v1.153.0).
|
|
141
|
+
Sending the same sample twice is therefore harmless.
|
|
142
|
+
|
|
143
|
+
**Latest sample.** `GET /stores/{store}/samples` needs `ifdd_id` and
|
|
144
|
+
`ifdd_property`; if more than one series has them (other name or unit), it
|
|
145
|
+
answers 422 and asks for `name` and `ifdd_unit`. It searches back from `at`
|
|
146
|
+
in growing windows (one hour, one day, 30 days, 366 days, then
|
|
147
|
+
`IFDD_TS_LOOKBACK_DAYS` or `max_age`), so a recent sample costs one small
|
|
148
|
+
read.
|
|
149
|
+
|
|
150
|
+
## Checks against the vocabulary
|
|
151
|
+
|
|
152
|
+
Before writing, every sample is checked against the vocabulary of its
|
|
153
|
+
property in the type service:
|
|
154
|
+
|
|
155
|
+
1. `ifdd_property` must be below `IFDD_TYPE_URL`. Its namespace is the IRI up
|
|
156
|
+
to the last `/` or `#`; the type service answers the namespace IRI with
|
|
157
|
+
the vocabulary.
|
|
158
|
+
2. The vocabulary must define the property as an `rdf:Property`.
|
|
159
|
+
3. If the property lists `qudt:applicableUnit`, `ifdd_unit` must be one of
|
|
160
|
+
them.
|
|
161
|
+
4. If it has `sh:minInclusive` or `sh:maxInclusive`, the value must lie
|
|
162
|
+
within.
|
|
163
|
+
|
|
164
|
+
The vocabulary is read with a JSON-LD processor, so prefixes and term names
|
|
165
|
+
do not matter; its context must be inline. A vocabulary is kept for
|
|
166
|
+
`IFDD_TS_VOCABULARY_CACHE_TTL` seconds; failures are not kept. A batch is
|
|
167
|
+
checked completely before anything is written; the 422 answer names every
|
|
168
|
+
bad sample by its index.
|
|
169
|
+
|
|
170
|
+
## Relation to ifdd-be-core
|
|
171
|
+
|
|
172
|
+
The token check and the documentation pages are copies of the ones in
|
|
173
|
+
`ifdd-be-core`, by decision (a shared package would have made this one
|
|
174
|
+
depend on PostgreSQL, rdflib and the backend settings). Fixes in one belong
|
|
175
|
+
in the other. If a third service needs them, they move into a package of
|
|
176
|
+
their own. One difference on purpose: a cached token is checked for expiry
|
|
177
|
+
on every use.
|
|
178
|
+
|
|
179
|
+
## Development
|
|
180
|
+
|
|
181
|
+
```bash
|
|
182
|
+
pip install -e '.[dev]'
|
|
183
|
+
pytest
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
`tests/test_vm_integration.py` runs against a real VictoriaMetrics when
|
|
187
|
+
`IFDD_TEST_VM_URL` is set; start it with deduplication like ifdd-ts:
|
|
188
|
+
|
|
189
|
+
```bash
|
|
190
|
+
victoria-metrics-prod -storageDataPath=/tmp/vm -httpListenAddr=127.0.0.1:8428 \
|
|
191
|
+
-dedup.minScrapeInterval=1ms -retentionPeriod=10y &
|
|
192
|
+
IFDD_TEST_VM_URL=http://127.0.0.1:8428 pytest
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
The CI does the same with the release binary v1.153.0 (SHA-256 checked).
|
|
196
|
+
|
|
197
|
+
Documentation: `docs/en` and `docs/de` (Sphinx, MyST);
|
|
198
|
+
`python tests/check_docs.py . --prefix ifdd-ts-core-` after exporting the
|
|
199
|
+
OpenAPI document to `docs/<lang>/openapi.json` (see `api-http.md`).
|
|
200
|
+
|
|
201
|
+
## Third-party files
|
|
202
|
+
|
|
203
|
+
`src/ifdd/ts/core/docs_assets/` holds unchanged copies of these npm packages,
|
|
204
|
+
the same as in ifdd-be-core; `docs_assets/assets.json` names version,
|
|
205
|
+
tarball, npm integrity and the SHA-256 of every file, and
|
|
206
|
+
`tests/test_docs_assets.py` checks them.
|
|
207
|
+
|
|
208
|
+
| Package | Version | License |
|
|
209
|
+
|---|---|---|
|
|
210
|
+
| `swagger-ui-dist` | 5.33.0 | Apache-2.0 (with `NOTICE`) |
|
|
211
|
+
| `redoc` | 2.5.4 | MIT |
|
|
212
|
+
|
|
213
|
+
The license texts are shipped next to the files and in the wheel; the bundles
|
|
214
|
+
contain code under further licenses (see the `*.LICENSE.txt` files).
|
|
215
|
+
`docs_assets/ifdd/` is part of this package (BSD-3-Clause). Update them
|
|
216
|
+
together with ifdd-be-core, as described in its README.
|
|
217
|
+
|
|
218
|
+
## AI assistance
|
|
219
|
+
|
|
220
|
+
Parts of this repository, especially tests, test data and documentation,
|
|
221
|
+
were developed or improved with the help of large language models. The
|
|
222
|
+
maintainer reviewed and approved every change.
|
|
223
|
+
|
|
224
|
+
## License
|
|
225
|
+
|
|
226
|
+
BSD-3-Clause, see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "ifdd-ts-core"
|
|
7
|
+
version = "26.10.2"
|
|
8
|
+
description = "Shared runtime of the IFDD time series API (ifdd-ts-api)"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
license = "BSD-3-Clause"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
|
|
14
|
+
dependencies = [
|
|
15
|
+
"fastapi~=0.142.2",
|
|
16
|
+
"uvicorn[standard]~=0.54.0",
|
|
17
|
+
"pydantic~=2.13.5",
|
|
18
|
+
"pydantic-settings~=2.15.0",
|
|
19
|
+
"httpx~=0.28.1",
|
|
20
|
+
"pyld~=3.3.0",
|
|
21
|
+
"cachetools~=7.2.1",
|
|
22
|
+
"PyJWT[crypto]~=2.15.1",
|
|
23
|
+
]
|
|
24
|
+
|
|
25
|
+
[project.optional-dependencies]
|
|
26
|
+
dev = [
|
|
27
|
+
"pytest~=9.1.1",
|
|
28
|
+
"pytest-asyncio~=1.4.0",
|
|
29
|
+
"pytest-cov",
|
|
30
|
+
"httpx2",
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
[project.urls]
|
|
34
|
+
Repository = "https://github.com/ifdd-de/ts-core"
|
|
35
|
+
|
|
36
|
+
[tool.setuptools.packages.find]
|
|
37
|
+
where = ["src"]
|
|
38
|
+
namespaces = true
|
|
39
|
+
|
|
40
|
+
# Swagger UI, ReDoc and the start script of the /docs and /redoc pages
|
|
41
|
+
# (ifdd.ts.core.docs_ui); the wheel must carry them.
|
|
42
|
+
[tool.setuptools.package-data]
|
|
43
|
+
"ifdd.ts.core" = ["docs_assets/*.json", "docs_assets/*/*"]
|
|
44
|
+
|
|
45
|
+
[tool.pytest.ini_options]
|
|
46
|
+
asyncio_mode = "auto"
|
|
47
|
+
testpaths = ["tests"]
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"""ifdd.ts.core - shared runtime of the IFDD time series API."""
|
|
2
|
+
|
|
3
|
+
from .app import create_app
|
|
4
|
+
from .config import CoreSettings, configure, get_settings
|
|
5
|
+
from .version import __version__
|
|
6
|
+
|
|
7
|
+
__all__ = [
|
|
8
|
+
"create_app",
|
|
9
|
+
"CoreSettings",
|
|
10
|
+
"configure",
|
|
11
|
+
"get_settings",
|
|
12
|
+
"__version__",
|
|
13
|
+
]
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
"""Application factory.
|
|
2
|
+
|
|
3
|
+
The API is served below the path prefix ``IFDD_TS_API_PATH`` (``/tsapi`` by
|
|
4
|
+
default), so that it shares the host of Grafana behind the proxy without a
|
|
5
|
+
rewrite: the proxy passes ``/tsapi/...`` unchanged, and the application
|
|
6
|
+
mounts its routes there. Inside, every path is relative to the prefix (the
|
|
7
|
+
OpenAPI document names the prefix as its server).
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from contextlib import asynccontextmanager
|
|
13
|
+
from typing import Iterable, Optional
|
|
14
|
+
|
|
15
|
+
from fastapi import FastAPI
|
|
16
|
+
from fastapi.routing import APIRouter
|
|
17
|
+
|
|
18
|
+
from .config import CoreSettings, configure, get_settings
|
|
19
|
+
from .docs_ui import install_docs
|
|
20
|
+
from .errors import install_error_handlers
|
|
21
|
+
from .http import close_client
|
|
22
|
+
from .logging import setup_logger
|
|
23
|
+
from .openapi import build_openapi
|
|
24
|
+
from .routers import meta_router, samples_router
|
|
25
|
+
from .version import __version__
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def create_app(
|
|
29
|
+
settings: Optional[CoreSettings] = None,
|
|
30
|
+
extra_routers: Optional[Iterable[APIRouter]] = None,
|
|
31
|
+
) -> FastAPI:
|
|
32
|
+
"""Create the application.
|
|
33
|
+
|
|
34
|
+
Args:
|
|
35
|
+
settings: Settings of the instance; registered with
|
|
36
|
+
:func:`~ifdd.ts.core.config.configure`. Without them the settings
|
|
37
|
+
are read from the environment.
|
|
38
|
+
extra_routers: Instance-specific endpoints, included after the
|
|
39
|
+
generic ones.
|
|
40
|
+
|
|
41
|
+
Returns:
|
|
42
|
+
The application for uvicorn (``uvicorn --factory
|
|
43
|
+
ifdd.ts.core:create_app`` or an instance's ``app.main:app``).
|
|
44
|
+
"""
|
|
45
|
+
if settings is not None:
|
|
46
|
+
configure(settings)
|
|
47
|
+
settings = get_settings()
|
|
48
|
+
logger = setup_logger(__name__)
|
|
49
|
+
logger.info(
|
|
50
|
+
f"Starting {settings.API_TITLE} {settings.IFDD_TS_API_VERSION} "
|
|
51
|
+
f"(ifdd-ts-core {__version__}) at {settings.IFDD_TS_API_PATH or '/'}"
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
@asynccontextmanager
|
|
55
|
+
async def lifespan(_: FastAPI):
|
|
56
|
+
"""Close the HTTP client when the application stops."""
|
|
57
|
+
yield
|
|
58
|
+
await close_client()
|
|
59
|
+
|
|
60
|
+
api = FastAPI(
|
|
61
|
+
title=settings.API_TITLE,
|
|
62
|
+
version=settings.IFDD_TS_API_VERSION,
|
|
63
|
+
docs_url=None,
|
|
64
|
+
redoc_url=None,
|
|
65
|
+
swagger_ui_oauth2_redirect_url=None,
|
|
66
|
+
root_path_in_servers=False,
|
|
67
|
+
lifespan=None if settings.IFDD_TS_API_PATH else lifespan,
|
|
68
|
+
)
|
|
69
|
+
install_error_handlers(api)
|
|
70
|
+
install_docs(api, settings.API_TITLE)
|
|
71
|
+
api.include_router(meta_router)
|
|
72
|
+
api.include_router(samples_router)
|
|
73
|
+
for router in extra_routers or ():
|
|
74
|
+
api.include_router(router)
|
|
75
|
+
api.openapi = build_openapi(api, settings)
|
|
76
|
+
|
|
77
|
+
if not settings.IFDD_TS_API_PATH:
|
|
78
|
+
return api
|
|
79
|
+
# A mounted application gets no lifespan events; the outer one runs them.
|
|
80
|
+
outer = FastAPI(docs_url=None, redoc_url=None, openapi_url=None, lifespan=lifespan)
|
|
81
|
+
outer.mount(settings.IFDD_TS_API_PATH, api)
|
|
82
|
+
outer.state.api = api
|
|
83
|
+
return outer
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def openapi_document(settings: Optional[CoreSettings] = None) -> dict:
|
|
87
|
+
"""The OpenAPI document of the API, e.g. for a documentation build.
|
|
88
|
+
|
|
89
|
+
Args:
|
|
90
|
+
settings: As for :func:`create_app`.
|
|
91
|
+
|
|
92
|
+
Returns:
|
|
93
|
+
The document as served at ``<IFDD_TS_API_PATH>/openapi.json``.
|
|
94
|
+
"""
|
|
95
|
+
app = create_app(settings)
|
|
96
|
+
return getattr(app.state, "api", app).openapi()
|