data-olympus 0.4.2__tar.gz → 0.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {data_olympus-0.4.2 → data_olympus-0.5.0}/CHANGELOG.md +55 -1
- {data_olympus-0.4.2 → data_olympus-0.5.0}/PKG-INFO +4 -1
- {data_olympus-0.4.2 → data_olympus-0.5.0}/SPEC.md +9 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/pyproject.toml +12 -1
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/cli/main.py +2 -0
- data_olympus-0.5.0/src/data_olympus/cli/migrate_cmd.py +105 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/config.py +47 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/health.py +11 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/index.py +40 -1
- data_olympus-0.5.0/src/data_olympus/metrics.py +223 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/models.py +11 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/refresh.py +50 -3
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/rest_api.py +36 -1
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/server.py +107 -1
- data_olympus-0.5.0/src/data_olympus/status_migrate.py +151 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/tools_enforce.py +34 -12
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/tools_read.py +6 -0
- data_olympus-0.5.0/src/data_olympus/version_check.py +89 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/.gitignore +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/LICENSE +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/NOTICE +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/README.md +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/benchmarks/README.md +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/bin/_kb_detect_workspace.sh +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/bin/_kb_enforce.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/bin/_kb_fallback.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/bin/kb +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/bin/kb-enforce-hook +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/bin/kb-session-recap-hook +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/bin/opencode/data-olympus-gate.ts +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/deploy/k8s/README.md +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/deploy/k8s/read-replica/README.md +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/docs/releases/README.md +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/example-bundle/projects/acme-app/README.md +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/example-bundle/projects/acme-app/components/api/README.md +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/__init__.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/audit_log.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/audit_trailers.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/auth.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/cli/__init__.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/cli/import_cmd.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/cli/indexgen.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/cli/init_cmd.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/cli/report_cmd.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/cli/validity_report_cmd.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/cli/verify_cmd.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/cooccurrence.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/dedup.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/durable.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/embeddings.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/enforce_policy.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/format/__init__.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/format/document.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/format/frontmatter.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/format/lint.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/format/validate.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/git_ops.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/governed_lane.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/importer/__init__.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/importer/adr.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/importer/flat.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/importer/model.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/importer/okf.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/importer/run.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/importer/stamp.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/maintenance.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/markdown_parse.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/onboarding.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/onboarding_inflight.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/onboarding_playbook.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/pending.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/principals.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/prompts.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/push_queue.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/query_expansion.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/rate_limit.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/report.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/safe_id.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/scaffold.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/search_gate.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/search_shortcut.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/session_metrics.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/setup_wizard.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/thin_pointer.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/tools_audit.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/tools_onboarding.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/tools_recap.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/tools_write.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/trigram.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/viewer/__init__.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/viewer/generator.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/worktrees.py +0 -0
- {data_olympus-0.4.2 → data_olympus-0.5.0}/src/data_olympus/write_gate.py +0 -0
|
@@ -12,6 +12,59 @@ This project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
|
12
12
|
|
|
13
13
|
## [Unreleased]
|
|
14
14
|
|
|
15
|
+
## [0.5.0] - 2026-07-12
|
|
16
|
+
|
|
17
|
+
### Added
|
|
18
|
+
|
|
19
|
+
- **Version-immutability release guard (`scripts/check_version_free.py`).** A
|
|
20
|
+
published version (PyPI, the ghcr `:vX.Y.Z` image tag, or a GitHub
|
|
21
|
+
release/tag) is immutable, so the release process now refuses to republish an
|
|
22
|
+
existing version. The guard runs as the first step of `tag-release.yml`'s
|
|
23
|
+
`decide` job (before the tag is pushed, so a duplicate never leaves an
|
|
24
|
+
orphaned public tag) and in PR CI (to catch a duplicate declared version
|
|
25
|
+
before merge). A legitimate reconcile (local tag `vX.Y.Z` already at HEAD) is
|
|
26
|
+
allowed and skips the registry lookups; a registry outage fails closed. The
|
|
27
|
+
operator can override a stuck registry with `KB_BYPASS_VERSION_CHECK=1`.
|
|
28
|
+
(gh #148 / KNA-67)
|
|
29
|
+
- **First-class reverse-proxy Host allowlist (`KB_PUBLIC_HOSTNAMES`).** New
|
|
30
|
+
config knob mapped onto fastmcp's Host-header allowlist when the HTTP app is
|
|
31
|
+
built, so operators configure data-olympus directly instead of reaching for
|
|
32
|
+
the dependency's `FASTMCP_HTTP_ALLOWED_HOSTS` env var (both are honoured and
|
|
33
|
+
merged). The server now emits a startup WARN when Host protection is on, it
|
|
34
|
+
binds a non-loopback address, and no public hostname is allowed: the exact
|
|
35
|
+
silent-breakage shape (readiness green, every proxied request 421) behind the
|
|
36
|
+
2026-07-09 kn-dev outage. A regression test boots the app with a configured
|
|
37
|
+
public hostname and asserts the good Host reaches the route while a foreign
|
|
38
|
+
Host is answered 421. (gh #139 / KNA-70)
|
|
39
|
+
- **Prometheus `/metrics` endpoint (optional `metrics` extra).**
|
|
40
|
+
`prometheus-client` is an OPTIONAL dependency; with it installed, `GET
|
|
41
|
+
/metrics` serves the standard exposition format (pending/push queue depths,
|
|
42
|
+
frozen count, push-failure counter, staleness, live sessions, per-tool call
|
|
43
|
+
counters, index-build duration + last-build timestamp). Without the extra the
|
|
44
|
+
route returns `501` and every metric update inside the core background loops
|
|
45
|
+
is a silent no-op, so a standard deployment is unaffected. (gh #69 / KNA-71)
|
|
46
|
+
- **Cached update-available signal.** The server now checks PyPI/GitHub for a
|
|
47
|
+
newer published `data-olympus` version in a background task and surfaces
|
|
48
|
+
`latest_version` / `update_available` in `kb_health` and
|
|
49
|
+
`GET /api/v1/health`. The health request path reads the cache only, and
|
|
50
|
+
compact health includes `latest_version` only when `update_available` is true.
|
|
51
|
+
Air-gapped deployments can set `KB_DISABLE_VERSION_CHECK=on` to make zero
|
|
52
|
+
outbound version-check calls. (gh #146 / KNA-68)
|
|
53
|
+
- **Legacy corpus status migration.** Status-less documents from pre-0.4.0
|
|
54
|
+
corpora now default to `active` in the SQLite index when
|
|
55
|
+
`KB_STATUS_AUTOFILL=on` (the default), preserving in-force retrieval without
|
|
56
|
+
mutating source markdown during index builds. Operators can persist the same
|
|
57
|
+
default explicitly with the idempotent `data-olympus migrate status --apply`
|
|
58
|
+
command. (gh #147 / KNA-69)
|
|
59
|
+
|
|
60
|
+
### Fixed
|
|
61
|
+
|
|
62
|
+
- **`kb_consult` demoted_writes CTA count.** The maintenance CTA now counts only
|
|
63
|
+
the session's live pending entries and is omitted once none remain, instead of
|
|
64
|
+
reporting stale "N write(s) awaiting operator review" after every demoted entry
|
|
65
|
+
had been resolved (the demoted_writes / pending_actions surface shipped since
|
|
66
|
+
v0.4.0 via #131). (gh #137 / KNA-72)
|
|
67
|
+
|
|
15
68
|
## [0.4.2] - 2026-07-11
|
|
16
69
|
|
|
17
70
|
### Added
|
|
@@ -1551,7 +1604,8 @@ This project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
|
1551
1604
|
- `docs/adoption.md`: bring-your-own-KB guide (author, lint, index, serve, wire an agent).
|
|
1552
1605
|
- `docs/comparison.md`: how data-olympus relates to OKF, enterprise catalogs, markdown KB tools, agent-context conventions, RAG, and ADR tooling.
|
|
1553
1606
|
|
|
1554
|
-
[Unreleased]: https://github.com/knaisoma/data-olympus/compare/v0.
|
|
1607
|
+
[Unreleased]: https://github.com/knaisoma/data-olympus/compare/v0.5.0...HEAD
|
|
1608
|
+
[0.5.0]: https://github.com/knaisoma/data-olympus/compare/v0.4.2...v0.5.0
|
|
1555
1609
|
[0.4.1]: https://github.com/knaisoma/data-olympus/compare/v0.4.0...v0.4.1
|
|
1556
1610
|
[0.4.0]: https://github.com/knaisoma/data-olympus/compare/v0.3.5...v0.4.0
|
|
1557
1611
|
[0.3.5]: https://github.com/knaisoma/data-olympus/compare/v0.3.4...v0.3.5
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: data-olympus
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.5.0
|
|
4
4
|
Summary: Governance-grade knowledge-base format (OKF-compatible) plus CLI and single-writer MCP server
|
|
5
5
|
Project-URL: Homepage, https://github.com/knaisoma/data-olympus
|
|
6
6
|
Project-URL: Repository, https://github.com/knaisoma/data-olympus
|
|
@@ -33,6 +33,7 @@ Provides-Extra: dev
|
|
|
33
33
|
Requires-Dist: fastembed<0.9,>=0.8; extra == 'dev'
|
|
34
34
|
Requires-Dist: httpx>=0.27; extra == 'dev'
|
|
35
35
|
Requires-Dist: mypy>=1.13; extra == 'dev'
|
|
36
|
+
Requires-Dist: prometheus-client>=0.20; extra == 'dev'
|
|
36
37
|
Requires-Dist: pytest-asyncio>=0.24; extra == 'dev'
|
|
37
38
|
Requires-Dist: pytest-cov>=5.0; extra == 'dev'
|
|
38
39
|
Requires-Dist: pytest>=8.3; extra == 'dev'
|
|
@@ -41,6 +42,8 @@ Requires-Dist: types-pyyaml>=6.0; extra == 'dev'
|
|
|
41
42
|
Requires-Dist: types-regex>=2024.5; extra == 'dev'
|
|
42
43
|
Provides-Extra: embeddings
|
|
43
44
|
Requires-Dist: fastembed<0.9,>=0.8; extra == 'embeddings'
|
|
45
|
+
Provides-Extra: metrics
|
|
46
|
+
Requires-Dist: prometheus-client>=0.20; extra == 'metrics'
|
|
44
47
|
Description-Content-Type: text/markdown
|
|
45
48
|
|
|
46
49
|
# data-olympus
|
|
@@ -281,6 +281,13 @@ The serving transport MUST be streamable HTTP (not stdio). The MCP endpoint is `
|
|
|
281
281
|
|
|
282
282
|
**Readiness probes MUST NOT target `/api/v1/health`.** A load balancer or orchestrator readiness check MUST use `GET /readyz` (200 once the process is up and the index is loaded, independent of data staleness). `/api/v1/health` keeps a 503-on-degraded contract for data-freshness alerting; wiring it as a readiness probe would eject a healthy single replica from its Service on a transient git-remote staleness. See [`docs/serving.md`](docs/serving.md).
|
|
283
283
|
|
|
284
|
+
**Update-available signal.** A server MAY surface cached `latest_version` and
|
|
285
|
+
`update_available` fields in `kb_health` / `GET /api/v1/health` when a newer
|
|
286
|
+
data-olympus package is published. These are informational serving-layer fields
|
|
287
|
+
only: they MUST NOT affect `degraded` or readiness, and the health request path
|
|
288
|
+
MUST read a cache rather than perform outbound registry calls. Deployments MAY
|
|
289
|
+
disable the check entirely for air-gapped operation.
|
|
290
|
+
|
|
284
291
|
**Enforcement endpoints.** A server MAY expose an enforcement surface that turns the advisory KB into a gated consultation proxy for code and architectural decisions. The endpoints are:
|
|
285
292
|
|
|
286
293
|
- `POST /api/v1/consult`: record a consultation for a `(source_session, workspace)` pair and return the governing rules for the supplied intent.
|
|
@@ -351,6 +358,8 @@ The `spec_version` field is optional in bundles targeting this `0.1` draft. It b
|
|
|
351
358
|
|
|
352
359
|
**Issue #114 (write-path `status` enforcement) does NOT bump this version.** `status` has been a required field under section 4.2 — and a `kb lint` error when absent — since the `0.1` draft; no prior version of this document ever described it as recommended, so there is no required-field-semantics change to record here. What issue #114 adds is a reference-implementation write-path check (section 8): a NEWLY created document missing `status` is now rejected at commit time, closing a gap where the write path was more permissive than `kb lint`. This does not affect any already-conformant bundle (which by definition already has `status` on every document) and does not change how an existing bundle is read, parsed, or validated — only how a brand-new document is admitted through the write path. Per the minor/major criteria above, that is neither: it is a reference-implementation enforcement-timing change, not a format change, so it is documented here rather than as a version increment. A bundle that is already non-conformant (missing `status` on one or more legacy documents) is unaffected in how it is READ — those documents keep being served, just never as in-force — see the maintenance ledger migration path in [`docs/operations.md`](docs/operations.md).
|
|
353
360
|
|
|
361
|
+
**Issue #147 / KNA-69 (legacy status autofill) also does NOT bump this version.** `#147` is a reference-implementation read-side default, not a format change: it does not alter what `status` means, its controlled vocabulary, or the requirement that documents carry it. It changes how the reference implementation treats a legacy document that is missing `status`. Under `#114` such a document was served but never in-force, so a pre-0.4.0 corpus silently lost its in-force guidance on upgrade. Under `#147` the index build treats a missing `status` as `active` by default (`KB_STATUS_AUTOFILL=on`), so the corpus keeps governing exactly as it did before the upgrade. This intentionally reverses the conservative "never guess status" stance `#114` took, and it does so for one deliberate reason: for the narrow legacy-upgrade case, a seamless default that preserves prior in-force behavior is worth more than conservatively flagging every legacy document as not-in-force and blocking it until a manual fix lands. The reversal is bounded and reversible: it applies only to documents with NO `status` (an explicit status is always honored verbatim), never rewrites the markdown source at index time (the build stays a read-only parse; the physical gap is still tracked by the maintenance ledger so the `missing_status` CTA keeps nagging), and is fully disabled by `KB_STATUS_AUTOFILL=off` for a deployment that prefers explicit flagging. Persisting the default into the markdown source is a separate, explicit operator action (`data-olympus migrate status --apply`), never an implicit side effect of indexing. See [`docs/operations.md`](docs/operations.md) section 5.2.
|
|
362
|
+
|
|
354
363
|
---
|
|
355
364
|
|
|
356
365
|
## 11. Relationship to other formats
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "data-olympus"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.5.0"
|
|
4
4
|
description = "Governance-grade knowledge-base format (OKF-compatible) plus CLI and single-writer MCP server"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.13"
|
|
@@ -46,6 +46,14 @@ Changelog = "https://github.com/knaisoma/data-olympus/blob/main/CHANGELOG.md"
|
|
|
46
46
|
embeddings = [
|
|
47
47
|
"fastembed>=0.8,<0.9",
|
|
48
48
|
]
|
|
49
|
+
# Optional Prometheus /metrics endpoint (issue #69). NOT a core dependency: the
|
|
50
|
+
# metrics registry is a no-op shim when prometheus-client is absent (its inc/
|
|
51
|
+
# set/observe calls live inside core background loops and tool wrappers, so a
|
|
52
|
+
# bare import must never crash a standard deployment). With the extra installed,
|
|
53
|
+
# /metrics serves the exposition format; without it, /metrics returns 501.
|
|
54
|
+
metrics = [
|
|
55
|
+
"prometheus-client>=0.20",
|
|
56
|
+
]
|
|
49
57
|
dev = [
|
|
50
58
|
"pytest>=8.3",
|
|
51
59
|
"pytest-asyncio>=0.24",
|
|
@@ -59,6 +67,9 @@ dev = [
|
|
|
59
67
|
# runtime; the tests opt in via KB_EMBEDDINGS_MODE and skip if the model
|
|
60
68
|
# cannot be fetched in the sandbox).
|
|
61
69
|
"fastembed>=0.8,<0.9",
|
|
70
|
+
# Exercise the Prometheus /metrics path in CI (the exposition-format test
|
|
71
|
+
# skips when this is absent; the no-op-shim tests run regardless).
|
|
72
|
+
"prometheus-client>=0.20",
|
|
62
73
|
]
|
|
63
74
|
bench = [
|
|
64
75
|
"tiktoken>=0.8",
|
|
@@ -143,6 +143,8 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
143
143
|
add_init_subparser(sub)
|
|
144
144
|
from data_olympus.cli.verify_cmd import add_verify_subparser
|
|
145
145
|
add_verify_subparser(sub)
|
|
146
|
+
from data_olympus.cli.migrate_cmd import add_migrate_subparser
|
|
147
|
+
add_migrate_subparser(sub)
|
|
146
148
|
return parser
|
|
147
149
|
|
|
148
150
|
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"""`data-olympus migrate status`: persist the status autofill to disk (#147).
|
|
2
|
+
|
|
3
|
+
The `status` subcommand writes the `status` field into legacy (pre-0.4.0)
|
|
4
|
+
markdown docs that are missing it, using the default `active` (see
|
|
5
|
+
``data_olympus.status_migrate``). This is the EXPLICIT counterpart to the
|
|
6
|
+
index build's VIRTUAL autofill (KB_STATUS_AUTOFILL): the build never touches
|
|
7
|
+
the markdown source, so an operator runs this once to make the change durable.
|
|
8
|
+
|
|
9
|
+
Without ``--apply`` the command is a DRY RUN: it reports which files would be
|
|
10
|
+
changed and exits 0, changing nothing. ``--apply`` performs the write and (when
|
|
11
|
+
an audit log path is configured) records one audited event per changed file.
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import os
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from typing import TYPE_CHECKING
|
|
18
|
+
|
|
19
|
+
if TYPE_CHECKING:
|
|
20
|
+
import argparse
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def add_migrate_subparser(
|
|
24
|
+
sub: argparse._SubParsersAction[argparse.ArgumentParser],
|
|
25
|
+
) -> None:
|
|
26
|
+
"""Wire the ``migrate`` command (with its ``status`` subcommand) in."""
|
|
27
|
+
p_migrate = sub.add_parser(
|
|
28
|
+
"migrate",
|
|
29
|
+
help="one-off corpus migrations (e.g. persist the status autofill)",
|
|
30
|
+
)
|
|
31
|
+
migrate_sub = p_migrate.add_subparsers(dest="migrate_command", required=True)
|
|
32
|
+
p_status = migrate_sub.add_parser(
|
|
33
|
+
"status",
|
|
34
|
+
help="write a default `status: active` into legacy docs missing it (#147)",
|
|
35
|
+
)
|
|
36
|
+
p_status.add_argument(
|
|
37
|
+
"path", nargs="?", default=".", help="corpus root (default: .)"
|
|
38
|
+
)
|
|
39
|
+
p_status.add_argument(
|
|
40
|
+
"--apply",
|
|
41
|
+
action="store_true",
|
|
42
|
+
help="write the change to disk (default: dry run, reports and changes nothing)",
|
|
43
|
+
)
|
|
44
|
+
p_status.add_argument(
|
|
45
|
+
"--audit-log",
|
|
46
|
+
default=None,
|
|
47
|
+
help="audit-log path for the applied writes (default: $KB_AUDIT_LOG_PATH; "
|
|
48
|
+
"omitted entirely if neither is set)",
|
|
49
|
+
)
|
|
50
|
+
p_status.set_defaults(func=_cmd_migrate_status)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _cmd_migrate_status(args: argparse.Namespace) -> int:
|
|
54
|
+
import sys
|
|
55
|
+
|
|
56
|
+
from data_olympus.format import discover_bundle_files
|
|
57
|
+
from data_olympus.format.document import Document
|
|
58
|
+
from data_olympus.status_migrate import DEFAULT_STATUS, apply_status_autofill
|
|
59
|
+
|
|
60
|
+
root = Path(args.path)
|
|
61
|
+
if not root.is_dir():
|
|
62
|
+
print(f"error: {root} is not a directory", file=sys.stderr)
|
|
63
|
+
return 2
|
|
64
|
+
|
|
65
|
+
if not args.apply:
|
|
66
|
+
# Dry run: report the files that WOULD be rewritten, change nothing.
|
|
67
|
+
would_change: list[str] = []
|
|
68
|
+
for md in discover_bundle_files(root):
|
|
69
|
+
try:
|
|
70
|
+
doc = Document.load(md)
|
|
71
|
+
except ValueError:
|
|
72
|
+
continue # malformed frontmatter: a lint concern, not migrated
|
|
73
|
+
status = doc.frontmatter.get("status")
|
|
74
|
+
if not (isinstance(status, str) and status.strip()):
|
|
75
|
+
would_change.append(str(md.relative_to(root)))
|
|
76
|
+
for rel in would_change:
|
|
77
|
+
print(f"would set status: {DEFAULT_STATUS} {rel}")
|
|
78
|
+
print(
|
|
79
|
+
f"dry run: {len(would_change)} document(s) would be updated "
|
|
80
|
+
f"(re-run with --apply to write)"
|
|
81
|
+
)
|
|
82
|
+
return 0
|
|
83
|
+
|
|
84
|
+
audit_log = None
|
|
85
|
+
audit_path = args.audit_log or os.environ.get("KB_AUDIT_LOG_PATH")
|
|
86
|
+
if audit_path:
|
|
87
|
+
from data_olympus.audit_log import AuditLog
|
|
88
|
+
|
|
89
|
+
audit_log = AuditLog(
|
|
90
|
+
log_path=audit_path,
|
|
91
|
+
hmac_key=os.environ.get("KB_AUDIT_HMAC_KEY", ""),
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
result = apply_status_autofill(root, audit_log=audit_log)
|
|
95
|
+
for rel in result.changed_paths:
|
|
96
|
+
print(f"set status: {DEFAULT_STATUS} {rel}")
|
|
97
|
+
if result.skipped_malformed:
|
|
98
|
+
for rel in result.skipped_malformed:
|
|
99
|
+
print(f"skipped (malformed frontmatter, fix with `kb lint`): {rel}")
|
|
100
|
+
print(
|
|
101
|
+
f"applied: {result.changed} changed, {result.already_present} already had "
|
|
102
|
+
f"status, {len(result.skipped_malformed)} skipped "
|
|
103
|
+
f"({result.scanned} scanned)"
|
|
104
|
+
)
|
|
105
|
+
return 0
|
|
@@ -75,6 +75,19 @@ class Config:
|
|
|
75
75
|
# Window sizes (days) for the "recently expired" / "expiring soon" buckets.
|
|
76
76
|
maintenance_recently_expired_days: int = 30
|
|
77
77
|
maintenance_expiring_soon_days: int = 30
|
|
78
|
+
# Virtual status autofill for legacy (pre-0.4.0) corpora (issue #147 / KNA-69).
|
|
79
|
+
# Default ON. When on, the index build treats a doc missing `status` as
|
|
80
|
+
# `active` IN MEMORY only (the SQLite `docs.status` column and the in-force
|
|
81
|
+
# retrieval view), so a pre-0.4.0 corpus does not silently lose its in-force
|
|
82
|
+
# docs on upgrade. The markdown source file is NEVER touched by the build:
|
|
83
|
+
# Index.build stays a read-only parse. The maintenance ledger still reports
|
|
84
|
+
# the PHYSICAL missing-status gap (so it keeps nagging until the operator runs
|
|
85
|
+
# `data-olympus migrate status --apply`, which is the only lane that writes
|
|
86
|
+
# `status` to disk). Set KB_STATUS_AUTOFILL=off to restore the conservative
|
|
87
|
+
# pre-#147 behavior (a status-less doc is served but never in-force). This
|
|
88
|
+
# intentionally reverses the #114 "never guess status" stance: for the narrow
|
|
89
|
+
# legacy-upgrade case, a seamless default beats conservative flagging.
|
|
90
|
+
status_autofill: bool = True
|
|
78
91
|
# Streamable-http session reaping: terminate transports idle beyond this many
|
|
79
92
|
# seconds to bound _server_instances (see session_metrics). 0 disables the
|
|
80
93
|
# reaper (observability-only). The scan runs every session_reap_interval_sec.
|
|
@@ -136,6 +149,27 @@ class Config:
|
|
|
136
149
|
# peer, ONLY safe when nothing untrusted can reach the port directly) to make
|
|
137
150
|
# the rate limiter see the true client IP behind the proxy. See docs/serving.md.
|
|
138
151
|
trusted_proxies: list[str] = field(default_factory=list)
|
|
152
|
+
# Public hostnames the reverse proxy presents in the Host header
|
|
153
|
+
# (KB_PUBLIC_HOSTNAMES, comma-separated). Mapped to fastmcp's Host-header
|
|
154
|
+
# allowlist when the HTTP app is built (see server.main). Empty (default)
|
|
155
|
+
# leaves fastmcp reading its own FASTMCP_HTTP_ALLOWED_HOSTS env knob, so
|
|
156
|
+
# behaviour is unchanged for existing deployments. This exists so operators
|
|
157
|
+
# configure data-olympus directly (`KB_*`) rather than reaching for the
|
|
158
|
+
# fastmcp dependency's env var. Getting it wrong is the silent-breakage shape
|
|
159
|
+
# that caused the 2026-07-09 kn-dev 421 outage: host-protection on + a public
|
|
160
|
+
# bind + no allowed host -> every proxied request 421s while readiness stays
|
|
161
|
+
# green (direct pod probes pass). See docs/serving.md and server._resolve_allowed_hosts.
|
|
162
|
+
public_hostnames: list[str] = field(default_factory=list)
|
|
163
|
+
# Periodic "a newer version is published" check (issue #146 / KNA-68).
|
|
164
|
+
# ``disable_version_check`` (KB_DISABLE_VERSION_CHECK, default off) makes an
|
|
165
|
+
# air-gapped deployment do ZERO outbound calls: the background task is not
|
|
166
|
+
# spawned at all. When enabled, the task runs once every
|
|
167
|
+
# ``version_check_interval_sec`` (default 24h) on a worker thread and caches
|
|
168
|
+
# the result on ServerState; the /api/v1/health route only READS the cached
|
|
169
|
+
# value, never the network (a blocking urllib lookup on the async request
|
|
170
|
+
# path would freeze the event loop for the timeout and stall readiness).
|
|
171
|
+
disable_version_check: bool = False
|
|
172
|
+
version_check_interval_sec: int = 86400
|
|
139
173
|
|
|
140
174
|
|
|
141
175
|
def _split_csv(raw: str) -> list[str]:
|
|
@@ -220,6 +254,10 @@ def load_config() -> Config:
|
|
|
220
254
|
maintenance_expiring_soon_days = int(
|
|
221
255
|
os.getenv("KB_MAINTENANCE_EXPIRING_SOON_DAYS", "30")
|
|
222
256
|
)
|
|
257
|
+
# KB_STATUS_AUTOFILL defaults to on (issue #147 / KNA-69): a legacy corpus
|
|
258
|
+
# missing `status` keeps its in-force docs after upgrade. Any explicit
|
|
259
|
+
# non-truthy value (off/0/false/no) restores the conservative behavior.
|
|
260
|
+
status_autofill = _env_bool(os.getenv("KB_STATUS_AUTOFILL", "on"))
|
|
223
261
|
session_idle_timeout_sec = int(os.getenv("KB_SESSION_IDLE_TIMEOUT_SEC", "300"))
|
|
224
262
|
session_reap_interval_sec = int(os.getenv("KB_SESSION_REAP_INTERVAL_SEC", "60"))
|
|
225
263
|
session_touch_interval_sec = int(os.getenv("KB_SESSION_TOUCH_INTERVAL_SEC", "30"))
|
|
@@ -250,6 +288,11 @@ def load_config() -> Config:
|
|
|
250
288
|
emb_enabled = _embeddings_enabled()
|
|
251
289
|
emb_cfg = _embeddings_config()
|
|
252
290
|
trusted_proxies = _split_csv(os.getenv("KB_TRUSTED_PROXIES", ""))
|
|
291
|
+
public_hostnames = _split_csv(os.getenv("KB_PUBLIC_HOSTNAMES", ""))
|
|
292
|
+
disable_version_check = _env_bool(os.getenv("KB_DISABLE_VERSION_CHECK", ""))
|
|
293
|
+
version_check_interval_sec = int(
|
|
294
|
+
os.getenv("KB_VERSION_CHECK_INTERVAL_SEC", "86400")
|
|
295
|
+
)
|
|
253
296
|
return Config(
|
|
254
297
|
kb_main_path=Path(os.environ.get("KB_MAIN_PATH", "/kb-main")),
|
|
255
298
|
kb_index_path=Path(os.environ.get("KB_INDEX_PATH", "/index/kb.db")),
|
|
@@ -285,6 +328,7 @@ def load_config() -> Config:
|
|
|
285
328
|
maintenance_ledger_path=maintenance_ledger_path,
|
|
286
329
|
maintenance_recently_expired_days=maintenance_recently_expired_days,
|
|
287
330
|
maintenance_expiring_soon_days=maintenance_expiring_soon_days,
|
|
331
|
+
status_autofill=status_autofill,
|
|
288
332
|
session_idle_timeout_sec=session_idle_timeout_sec,
|
|
289
333
|
session_reap_interval_sec=session_reap_interval_sec,
|
|
290
334
|
session_touch_interval_sec=session_touch_interval_sec,
|
|
@@ -302,4 +346,7 @@ def load_config() -> Config:
|
|
|
302
346
|
embeddings_weight=emb_cfg.weight,
|
|
303
347
|
embeddings_model=emb_cfg.model_name,
|
|
304
348
|
trusted_proxies=trusted_proxies,
|
|
349
|
+
public_hostnames=public_hostnames,
|
|
350
|
+
disable_version_check=disable_version_check,
|
|
351
|
+
version_check_interval_sec=version_check_interval_sec,
|
|
305
352
|
)
|
|
@@ -53,6 +53,13 @@ class HealthState:
|
|
|
53
53
|
# NOT flip ``degraded``; a superseded doc silently losing its own status
|
|
54
54
|
# flip is a governance data-quality signal, not a service failure.
|
|
55
55
|
graph_excluded_docs: int = 0
|
|
56
|
+
# Latest published version cached by the periodic version-check background
|
|
57
|
+
# task (issue #146 / KNA-68). None until the first check completes, or
|
|
58
|
+
# forever when the check is disabled (KB_DISABLE_VERSION_CHECK) or offline.
|
|
59
|
+
# The health route only READS this cache; it never touches the network, so a
|
|
60
|
+
# slow/unreachable PyPI can never stall the async request path.
|
|
61
|
+
latest_version: str | None = None
|
|
62
|
+
update_available: bool = False
|
|
56
63
|
|
|
57
64
|
|
|
58
65
|
def snapshot(
|
|
@@ -74,6 +81,8 @@ def snapshot(
|
|
|
74
81
|
last_successful_refresh_at: float | None = None,
|
|
75
82
|
remote_head_sha: str | None = None,
|
|
76
83
|
live_sessions: int | None = None,
|
|
84
|
+
latest_version: str | None = None,
|
|
85
|
+
update_available: bool = False,
|
|
77
86
|
) -> HealthState:
|
|
78
87
|
"""Compose a HealthState from the index and the last-pull/push timestamps.
|
|
79
88
|
|
|
@@ -127,4 +136,6 @@ def snapshot(
|
|
|
127
136
|
malformed_frontmatter=malformed_frontmatter,
|
|
128
137
|
malformed_validity=malformed_validity,
|
|
129
138
|
graph_excluded_docs=graph_excluded_docs,
|
|
139
|
+
latest_version=latest_version,
|
|
140
|
+
update_available=update_available,
|
|
130
141
|
)
|
|
@@ -219,6 +219,15 @@ _DEFAULT_STATUS_WEIGHTS: dict[str, float] = {
|
|
|
219
219
|
# source (format.validate.IN_FORCE_STATUSES); do not hand-list statuses here.
|
|
220
220
|
_IN_FORCE_STATUS_LIST: tuple[str, ...] = tuple(sorted(IN_FORCE_STATUSES))
|
|
221
221
|
|
|
222
|
+
# Virtual status autofill (issue #147 / KNA-69). The default status written IN
|
|
223
|
+
# MEMORY into the SQLite `docs.status` column for a legacy doc that has no
|
|
224
|
+
# `status` field, when autofill is enabled for the index. `active` preserves the
|
|
225
|
+
# pre-0.4.0 in-force behavior (it is in format.validate.IN_FORCE_STATUSES). The
|
|
226
|
+
# markdown source file is untouched by the build; the same default is written to
|
|
227
|
+
# disk only by the explicit `data-olympus migrate status --apply` lane
|
|
228
|
+
# (data_olympus.status_migrate.DEFAULT_STATUS, kept in sync with this value).
|
|
229
|
+
DEFAULT_AUTOFILL_STATUS = "active"
|
|
230
|
+
|
|
222
231
|
# Generic, deployment-neutral default taxonomy. A deployment with a different
|
|
223
232
|
# directory layout supplies its own table at runtime via KB_TAXONOMY_PATH (a
|
|
224
233
|
# JSON file holding a list of [prefix, tier, category] triples). The two
|
|
@@ -647,8 +656,18 @@ class Index:
|
|
|
647
656
|
maintenance_ledger_path: str = DEFAULT_LEDGER_PATH,
|
|
648
657
|
maintenance_recently_expired_days: int = DEFAULT_RECENTLY_EXPIRED_DAYS,
|
|
649
658
|
maintenance_expiring_soon_days: int = DEFAULT_EXPIRING_SOON_DAYS,
|
|
659
|
+
status_autofill: bool = True,
|
|
650
660
|
) -> None:
|
|
651
661
|
self._db_path = db_path
|
|
662
|
+
# Virtual status autofill (issue #147 / KNA-69). When True (default), a
|
|
663
|
+
# doc missing `status` is indexed with `docs.status` = active IN MEMORY,
|
|
664
|
+
# so a pre-0.4.0 corpus keeps its in-force docs after upgrade without the
|
|
665
|
+
# build ever touching the markdown source. The physical missing-status
|
|
666
|
+
# gap is still reported by the maintenance ledger (DocAuditRow carries the
|
|
667
|
+
# PHYSICAL status), so the ledger keeps nagging until an operator runs
|
|
668
|
+
# `data-olympus migrate status --apply`. Off restores the conservative
|
|
669
|
+
# pre-#147 behavior (a status-less doc is served but never in-force).
|
|
670
|
+
self._status_autofill = status_autofill
|
|
652
671
|
# Embeddings (issue #42) are threaded in from Config, NOT re-read from env
|
|
653
672
|
# here (reviewer concern 2): Config (via load_config) is the single source
|
|
654
673
|
# of truth for KB_EMBEDDINGS_MODE/MODEL/WEIGHT. ``embeddings`` being non-
|
|
@@ -1090,6 +1109,11 @@ class Index:
|
|
|
1090
1109
|
"block was treated as absent for this doc",
|
|
1091
1110
|
rel,
|
|
1092
1111
|
)
|
|
1112
|
+
# The maintenance ledger audits the PHYSICAL corpus, so it records
|
|
1113
|
+
# the doc's real (pre-autofill) status: a legacy doc missing
|
|
1114
|
+
# `status` still counts as missing here, so the ledger keeps
|
|
1115
|
+
# nagging until an operator runs `migrate status --apply`, even
|
|
1116
|
+
# while virtual autofill (below) serves it as in-force.
|
|
1093
1117
|
maintenance_rows.append(
|
|
1094
1118
|
DocAuditRow(
|
|
1095
1119
|
path=str(rel), id=doc_id, status=doc.status,
|
|
@@ -1097,6 +1121,21 @@ class Index:
|
|
|
1097
1121
|
is_reserved=rel.name in RESERVED,
|
|
1098
1122
|
)
|
|
1099
1123
|
)
|
|
1124
|
+
# Virtual status autofill (issue #147 / KNA-69): a legacy doc
|
|
1125
|
+
# missing `status` is indexed as `active` IN MEMORY only (this
|
|
1126
|
+
# SQLite column, never the markdown source), so a pre-0.4.0 corpus
|
|
1127
|
+
# keeps its in-force docs after upgrade. Gated on the per-index
|
|
1128
|
+
# ``_status_autofill`` flag; off restores the conservative
|
|
1129
|
+
# served-but-never-in-force behavior. A reserved filename
|
|
1130
|
+
# (index.md/log.md/template.md) is exempt from the status schema
|
|
1131
|
+
# requirement, so it is NOT autofilled (it never needed a status).
|
|
1132
|
+
indexed_status = doc.status
|
|
1133
|
+
if (
|
|
1134
|
+
self._status_autofill
|
|
1135
|
+
and not doc.status
|
|
1136
|
+
and rel.name not in RESERVED
|
|
1137
|
+
):
|
|
1138
|
+
indexed_status = DEFAULT_AUTOFILL_STATUS
|
|
1100
1139
|
path_tier, path_category = _classify_by_path(str(rel), path_rules)
|
|
1101
1140
|
final_tier = doc.tier or path_tier
|
|
1102
1141
|
final_category = doc.category or path_category
|
|
@@ -1118,7 +1157,7 @@ class Index:
|
|
|
1118
1157
|
"valid_from, valid_until, last_verified, recheck_by, verification_source, "
|
|
1119
1158
|
"is_inbox) "
|
|
1120
1159
|
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
|
1121
|
-
(doc_id, str(rel), final_tier, final_category,
|
|
1160
|
+
(doc_id, str(rel), final_tier, final_category, indexed_status, doc.doc_type,
|
|
1122
1161
|
applies_when_str, doc.description, doc.title, tags_str, content_markdown,
|
|
1123
1162
|
last_modified, lm_source, doc.git_remote_url,
|
|
1124
1163
|
doc.valid_from or None, doc.valid_until or None,
|