okf-agents 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- okf_agents/__init__.py +39 -0
- okf_agents/_internal/__init__.py +1 -0
- okf_agents/_internal/graph_utils.py +86 -0
- okf_agents/_internal/parser.py +285 -0
- okf_agents/bundle.py +288 -0
- okf_agents/exceptions.py +66 -0
- okf_agents/indexing.py +172 -0
- okf_agents/models.py +106 -0
- okf_agents/navigator.py +407 -0
- okf_agents/py.typed +0 -0
- okf_agents/retriever.py +157 -0
- okf_agents/router.py +120 -0
- okf_agents/tools.py +228 -0
- okf_agents-0.1.0.dist-info/METADATA +456 -0
- okf_agents-0.1.0.dist-info/RECORD +17 -0
- okf_agents-0.1.0.dist-info/WHEEL +4 -0
- okf_agents-0.1.0.dist-info/licenses/LICENSE +21 -0
okf_agents/__init__.py
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"""okf-agents: Open Knowledge Format bundles for LangGraph agents."""
|
|
2
|
+
|
|
3
|
+
from okf_agents.bundle import OKFBundle
|
|
4
|
+
from okf_agents.exceptions import (
|
|
5
|
+
BundleNotFoundError,
|
|
6
|
+
BundleValidationError,
|
|
7
|
+
ConceptNotFoundError,
|
|
8
|
+
LinkResolutionError,
|
|
9
|
+
OKFError,
|
|
10
|
+
)
|
|
11
|
+
from okf_agents.indexing import sync_bundle_to_vector_store
|
|
12
|
+
from okf_agents.models import BundleIndex, Concept, ConceptFrontmatter, LinkEdge, SyncResult
|
|
13
|
+
from okf_agents.navigator import create_okf_navigator
|
|
14
|
+
from okf_agents.retriever import OKFGraphRetriever, OKFRetriever
|
|
15
|
+
from okf_agents.router import create_okf_router
|
|
16
|
+
from okf_agents.tools import create_okf_tools
|
|
17
|
+
|
|
18
|
+
__version__ = "0.1.0"
|
|
19
|
+
|
|
20
|
+
__all__ = [
|
|
21
|
+
"BundleIndex",
|
|
22
|
+
"BundleNotFoundError",
|
|
23
|
+
"BundleValidationError",
|
|
24
|
+
"Concept",
|
|
25
|
+
"ConceptFrontmatter",
|
|
26
|
+
"ConceptNotFoundError",
|
|
27
|
+
"LinkEdge",
|
|
28
|
+
"LinkResolutionError",
|
|
29
|
+
"OKFBundle",
|
|
30
|
+
"OKFError",
|
|
31
|
+
"OKFGraphRetriever",
|
|
32
|
+
"OKFRetriever",
|
|
33
|
+
"SyncResult",
|
|
34
|
+
"__version__",
|
|
35
|
+
"create_okf_navigator",
|
|
36
|
+
"create_okf_router",
|
|
37
|
+
"create_okf_tools",
|
|
38
|
+
"sync_bundle_to_vector_store",
|
|
39
|
+
]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Internal implementation details. Not part of the public API."""
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
"""Dependency-free directed-graph helpers for the bundle link graph.
|
|
2
|
+
|
|
3
|
+
Adjacency maps use concept IDs and hold deduplicated, ID-sorted neighbor
|
|
4
|
+
lists so every traversal is deterministic. No graph package is used.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from collections.abc import Iterable, Mapping, Sequence
|
|
10
|
+
|
|
11
|
+
from okf_agents.models import LinkEdge
|
|
12
|
+
|
|
13
|
+
__all__ = [
|
|
14
|
+
"breadth_first_reachable",
|
|
15
|
+
"build_adjacency",
|
|
16
|
+
"merge_adjacency",
|
|
17
|
+
]
|
|
18
|
+
|
|
19
|
+
Adjacency = dict[str, list[str]]
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def build_adjacency(edges: Iterable[LinkEdge]) -> tuple[Adjacency, Adjacency]:
|
|
23
|
+
"""Build ``(outbound, inbound)`` adjacency maps from resolved edges.
|
|
24
|
+
|
|
25
|
+
Unresolved edges are excluded because their targets are not loaded
|
|
26
|
+
concepts. Neighbor lists are deduplicated and sorted by concept ID.
|
|
27
|
+
"""
|
|
28
|
+
outbound: dict[str, set[str]] = {}
|
|
29
|
+
inbound: dict[str, set[str]] = {}
|
|
30
|
+
for edge in edges:
|
|
31
|
+
if not edge.resolved:
|
|
32
|
+
continue
|
|
33
|
+
outbound.setdefault(edge.source_id, set()).add(edge.target_id)
|
|
34
|
+
inbound.setdefault(edge.target_id, set()).add(edge.source_id)
|
|
35
|
+
return (
|
|
36
|
+
{node: sorted(targets) for node, targets in outbound.items()},
|
|
37
|
+
{node: sorted(sources) for node, sources in inbound.items()},
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def merge_adjacency(
|
|
42
|
+
first: Mapping[str, Sequence[str]],
|
|
43
|
+
second: Mapping[str, Sequence[str]],
|
|
44
|
+
) -> Adjacency:
|
|
45
|
+
"""Merge two adjacency maps into deduplicated, ID-sorted neighbor lists."""
|
|
46
|
+
merged: dict[str, set[str]] = {}
|
|
47
|
+
for adjacency in (first, second):
|
|
48
|
+
for node, neighbors in adjacency.items():
|
|
49
|
+
merged.setdefault(node, set()).update(neighbors)
|
|
50
|
+
return {node: sorted(neighbors) for node, neighbors in merged.items()}
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def breadth_first_reachable(
|
|
54
|
+
adjacency: Mapping[str, Sequence[str]],
|
|
55
|
+
start: str,
|
|
56
|
+
hops: int,
|
|
57
|
+
) -> list[str]:
|
|
58
|
+
"""Return node IDs reachable from ``start`` within ``hops`` edges.
|
|
59
|
+
|
|
60
|
+
Nodes are marked visited when enqueued, so cycles terminate and each
|
|
61
|
+
node appears at most once. ``start`` itself is excluded. Results are
|
|
62
|
+
ordered by distance from ``start``, then by ID within each distance.
|
|
63
|
+
|
|
64
|
+
Raises:
|
|
65
|
+
ValueError: If ``hops`` is negative.
|
|
66
|
+
"""
|
|
67
|
+
if hops < 0:
|
|
68
|
+
raise ValueError(f"hops must be non-negative, got {hops}")
|
|
69
|
+
visited = {start}
|
|
70
|
+
reachable: list[str] = []
|
|
71
|
+
frontier = [start]
|
|
72
|
+
for _ in range(hops):
|
|
73
|
+
next_frontier = sorted(
|
|
74
|
+
{
|
|
75
|
+
neighbor
|
|
76
|
+
for node in frontier
|
|
77
|
+
for neighbor in adjacency.get(node, ())
|
|
78
|
+
if neighbor not in visited
|
|
79
|
+
}
|
|
80
|
+
)
|
|
81
|
+
if not next_frontier:
|
|
82
|
+
break
|
|
83
|
+
visited.update(next_frontier)
|
|
84
|
+
reachable.extend(next_frontier)
|
|
85
|
+
frontier = next_frontier
|
|
86
|
+
return reachable
|
|
@@ -0,0 +1,285 @@
|
|
|
1
|
+
"""Pure parsing functions for OKF v0.1 concept files and bundle indexes.
|
|
2
|
+
|
|
3
|
+
Every function is side-effect free: no filesystem writes and no network
|
|
4
|
+
access. Invalid input raises
|
|
5
|
+
:class:`~okf_agents.exceptions.BundleValidationError` keyed by the
|
|
6
|
+
root-relative source path so the bundle loader can aggregate failures.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import posixpath
|
|
12
|
+
import re
|
|
13
|
+
from collections.abc import Sequence
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
import yaml
|
|
18
|
+
from pydantic import ValidationError
|
|
19
|
+
|
|
20
|
+
from okf_agents.exceptions import BundleValidationError
|
|
21
|
+
from okf_agents.models import BundleIndex, Concept, ConceptFrontmatter, LinkEdge
|
|
22
|
+
|
|
23
|
+
__all__ = [
|
|
24
|
+
"extract_internal_links",
|
|
25
|
+
"normalize_link_target",
|
|
26
|
+
"parse_bundle_index",
|
|
27
|
+
"parse_concept",
|
|
28
|
+
"parse_frontmatter",
|
|
29
|
+
"split_frontmatter",
|
|
30
|
+
"synthesize_bundle_index",
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
_FRONTMATTER_DELIMITER = "---"
|
|
34
|
+
_STANDARD_FRONTMATTER_KEYS = ("type", "title", "description", "resource", "tags", "timestamp")
|
|
35
|
+
_MD_SUFFIX = ".md"
|
|
36
|
+
|
|
37
|
+
# Standard inline links only: images are excluded by the lookbehind, and
|
|
38
|
+
# reference-style links and autolinks never match because they lack "(...)".
|
|
39
|
+
_INLINE_LINK_RE = re.compile(
|
|
40
|
+
r"(?<!!)\[(?P<anchor>[^\]]*)\]\((?P<target>[^()\s]+)(?:\s+\"[^\"]*\")?\)"
|
|
41
|
+
)
|
|
42
|
+
_URL_SCHEME_RE = re.compile(r"^[A-Za-z][A-Za-z0-9+.-]*:")
|
|
43
|
+
_FENCE_RE = re.compile(r"^ {0,3}(```|~~~)")
|
|
44
|
+
_H1_RE = re.compile(r"^# +(?P<title>.+?) *#* *$")
|
|
45
|
+
_LIST_ITEM_RE = re.compile(r"^(?:[-*+]|\d+[.)]) ")
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _invalid(source: str, reason: str) -> BundleValidationError:
|
|
49
|
+
return BundleValidationError({source: reason})
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def split_frontmatter(raw: str, *, source: str) -> tuple[dict[str, Any], str]:
|
|
53
|
+
"""Split raw concept text into its YAML frontmatter mapping and body.
|
|
54
|
+
|
|
55
|
+
The text must start with a ``---`` line and contain a matching closing
|
|
56
|
+
``---`` line; both delimiters accept LF or CRLF endings. The returned
|
|
57
|
+
body excludes the frontmatter and both delimiter lines, preserving the
|
|
58
|
+
original line endings.
|
|
59
|
+
|
|
60
|
+
Raises:
|
|
61
|
+
BundleValidationError: If a delimiter is missing, the YAML is
|
|
62
|
+
malformed, or the frontmatter is not a mapping.
|
|
63
|
+
"""
|
|
64
|
+
lines = raw.splitlines(keepends=True)
|
|
65
|
+
if not lines or lines[0].rstrip("\r\n") != _FRONTMATTER_DELIMITER:
|
|
66
|
+
raise _invalid(source, "missing opening '---' frontmatter delimiter")
|
|
67
|
+
close_index = next(
|
|
68
|
+
(
|
|
69
|
+
index
|
|
70
|
+
for index, line in enumerate(lines[1:], start=1)
|
|
71
|
+
if line.rstrip("\r\n") == _FRONTMATTER_DELIMITER
|
|
72
|
+
),
|
|
73
|
+
None,
|
|
74
|
+
)
|
|
75
|
+
if close_index is None:
|
|
76
|
+
raise _invalid(source, "missing closing '---' frontmatter delimiter")
|
|
77
|
+
yaml_text = "".join(lines[1:close_index])
|
|
78
|
+
body = "".join(lines[close_index + 1 :])
|
|
79
|
+
try:
|
|
80
|
+
mapping = yaml.safe_load(yaml_text)
|
|
81
|
+
except yaml.YAMLError as exc:
|
|
82
|
+
raise _invalid(source, f"malformed YAML frontmatter: {exc}") from exc
|
|
83
|
+
if mapping is None:
|
|
84
|
+
mapping = {}
|
|
85
|
+
if not isinstance(mapping, dict):
|
|
86
|
+
raise _invalid(source, "frontmatter must be a YAML mapping")
|
|
87
|
+
return mapping, body
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def parse_frontmatter(mapping: dict[str, Any], *, source: str) -> ConceptFrontmatter:
|
|
91
|
+
"""Validate a frontmatter mapping into a :class:`ConceptFrontmatter`.
|
|
92
|
+
|
|
93
|
+
Standard keys become typed fields; every unknown key is kept only in
|
|
94
|
+
``extra``.
|
|
95
|
+
|
|
96
|
+
Raises:
|
|
97
|
+
BundleValidationError: If ``type`` is missing or empty, or any
|
|
98
|
+
field has an invalid type.
|
|
99
|
+
"""
|
|
100
|
+
standard = {key: mapping[key] for key in _STANDARD_FRONTMATTER_KEYS if key in mapping}
|
|
101
|
+
extra = {key: value for key, value in mapping.items() if key not in _STANDARD_FRONTMATTER_KEYS}
|
|
102
|
+
if "type" not in standard:
|
|
103
|
+
raise _invalid(source, "frontmatter is missing required key 'type'")
|
|
104
|
+
try:
|
|
105
|
+
return ConceptFrontmatter(**standard, extra=extra)
|
|
106
|
+
except ValidationError as exc:
|
|
107
|
+
reasons = "; ".join(
|
|
108
|
+
f"{'.'.join(str(loc) for loc in error['loc'])}: {error['msg']}"
|
|
109
|
+
for error in exc.errors()
|
|
110
|
+
)
|
|
111
|
+
raise _invalid(source, f"invalid frontmatter: {reasons}") from exc
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def normalize_link_target(target: str, *, source_id: str, source: str) -> str | None:
|
|
115
|
+
"""Normalize an inline-link target to a concept ID, or return ``None``.
|
|
116
|
+
|
|
117
|
+
``None`` marks targets that are not internal Markdown links: external
|
|
118
|
+
URLs (any scheme or protocol-relative), fragment-only links, and
|
|
119
|
+
targets without a ``.md`` suffix. Fragments are stripped first.
|
|
120
|
+
Bundle-relative targets (``/path/file.md``) resolve from the bundle
|
|
121
|
+
root; all other targets resolve from the source concept's directory.
|
|
122
|
+
|
|
123
|
+
Raises:
|
|
124
|
+
BundleValidationError: If the target traverses outside the bundle.
|
|
125
|
+
"""
|
|
126
|
+
if _URL_SCHEME_RE.match(target) or target.startswith("//"):
|
|
127
|
+
return None
|
|
128
|
+
path_part = target.split("#", 1)[0]
|
|
129
|
+
if not path_part.endswith(_MD_SUFFIX):
|
|
130
|
+
return None
|
|
131
|
+
if path_part.startswith("/"):
|
|
132
|
+
candidate = posixpath.normpath(path_part.lstrip("/"))
|
|
133
|
+
else:
|
|
134
|
+
candidate = posixpath.normpath(posixpath.join(posixpath.dirname(source_id), path_part))
|
|
135
|
+
if candidate == ".." or candidate.startswith("../"):
|
|
136
|
+
raise _invalid(source, f"link target escapes the bundle: {target!r}")
|
|
137
|
+
return candidate[: -len(_MD_SUFFIX)]
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def extract_internal_links(body: str, *, source_id: str, source: str) -> list[LinkEdge]:
|
|
141
|
+
"""Extract internal inline Markdown links as unresolved edges.
|
|
142
|
+
|
|
143
|
+
Returns one :class:`LinkEdge` (``resolved=False``) per link occurrence
|
|
144
|
+
in document order, retaining repeats so callers can build a multigraph.
|
|
145
|
+
Image syntax, links inside fenced code blocks, and targets rejected by
|
|
146
|
+
:func:`normalize_link_target` are ignored.
|
|
147
|
+
|
|
148
|
+
Raises:
|
|
149
|
+
BundleValidationError: If any link target escapes the bundle.
|
|
150
|
+
"""
|
|
151
|
+
edges: list[LinkEdge] = []
|
|
152
|
+
fence_marker: str | None = None
|
|
153
|
+
for line in body.splitlines():
|
|
154
|
+
fence_match = _FENCE_RE.match(line)
|
|
155
|
+
if fence_match:
|
|
156
|
+
marker = fence_match.group(1)
|
|
157
|
+
if fence_marker is None:
|
|
158
|
+
fence_marker = marker
|
|
159
|
+
elif marker == fence_marker:
|
|
160
|
+
fence_marker = None
|
|
161
|
+
continue
|
|
162
|
+
if fence_marker is not None:
|
|
163
|
+
continue
|
|
164
|
+
for match in _INLINE_LINK_RE.finditer(line):
|
|
165
|
+
target_id = normalize_link_target(
|
|
166
|
+
match.group("target"), source_id=source_id, source=source
|
|
167
|
+
)
|
|
168
|
+
if target_id is not None:
|
|
169
|
+
edges.append(
|
|
170
|
+
LinkEdge(
|
|
171
|
+
source_id=source_id,
|
|
172
|
+
target_id=target_id,
|
|
173
|
+
anchor_text=match.group("anchor"),
|
|
174
|
+
)
|
|
175
|
+
)
|
|
176
|
+
return edges
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def parse_concept(raw: str, *, bundle_root: Path, file_path: Path) -> Concept:
|
|
180
|
+
"""Parse one concept file from raw text plus its bundle and file paths.
|
|
181
|
+
|
|
182
|
+
The returned model preserves ``raw`` verbatim, excludes the
|
|
183
|
+
frontmatter delimiters from ``body``, exposes ``path`` as the resolved
|
|
184
|
+
absolute path string, and deduplicates ``outbound_links`` in
|
|
185
|
+
first-seen order.
|
|
186
|
+
|
|
187
|
+
Raises:
|
|
188
|
+
BundleValidationError: Keyed by the root-relative path when the
|
|
189
|
+
frontmatter is invalid, a link escapes the bundle, or the file
|
|
190
|
+
lies outside the bundle root.
|
|
191
|
+
"""
|
|
192
|
+
resolved_root = bundle_root.resolve()
|
|
193
|
+
resolved_file = file_path if file_path.is_absolute() else resolved_root / file_path
|
|
194
|
+
resolved_file = resolved_file.resolve()
|
|
195
|
+
try:
|
|
196
|
+
relative = resolved_file.relative_to(resolved_root)
|
|
197
|
+
except ValueError as exc:
|
|
198
|
+
raise _invalid(str(file_path), "concept file lies outside the bundle root") from exc
|
|
199
|
+
source = relative.as_posix()
|
|
200
|
+
concept_id = source[: -len(_MD_SUFFIX)] if source.endswith(_MD_SUFFIX) else source
|
|
201
|
+
|
|
202
|
+
mapping, body = split_frontmatter(raw, source=source)
|
|
203
|
+
frontmatter = parse_frontmatter(mapping, source=source)
|
|
204
|
+
edges = extract_internal_links(body, source_id=concept_id, source=source)
|
|
205
|
+
outbound_links = list(dict.fromkeys(edge.target_id for edge in edges))
|
|
206
|
+
return Concept(
|
|
207
|
+
id=concept_id,
|
|
208
|
+
path=str(resolved_file),
|
|
209
|
+
frontmatter=frontmatter,
|
|
210
|
+
body=body,
|
|
211
|
+
outbound_links=outbound_links,
|
|
212
|
+
raw=raw,
|
|
213
|
+
)
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def _extract_title_and_description(markdown: str) -> tuple[str | None, str | None]:
|
|
217
|
+
"""Return the first H1 title and first non-heading paragraph, if any."""
|
|
218
|
+
title: str | None = None
|
|
219
|
+
paragraph: list[str] = []
|
|
220
|
+
fenced = False
|
|
221
|
+
for line in markdown.splitlines():
|
|
222
|
+
if _FENCE_RE.match(line):
|
|
223
|
+
if paragraph:
|
|
224
|
+
break
|
|
225
|
+
fenced = not fenced
|
|
226
|
+
continue
|
|
227
|
+
if fenced:
|
|
228
|
+
continue
|
|
229
|
+
stripped = line.strip()
|
|
230
|
+
if not stripped:
|
|
231
|
+
if paragraph:
|
|
232
|
+
break
|
|
233
|
+
continue
|
|
234
|
+
if stripped.startswith("#"):
|
|
235
|
+
if paragraph:
|
|
236
|
+
break
|
|
237
|
+
h1 = _H1_RE.match(stripped)
|
|
238
|
+
if h1 and title is None:
|
|
239
|
+
title = h1.group("title")
|
|
240
|
+
continue
|
|
241
|
+
if not paragraph and _LIST_ITEM_RE.match(stripped):
|
|
242
|
+
# A link list is not a descriptive paragraph.
|
|
243
|
+
continue
|
|
244
|
+
paragraph.append(stripped)
|
|
245
|
+
description = " ".join(paragraph) if paragraph else None
|
|
246
|
+
return title, description
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def parse_bundle_index(raw: str, *, source: str = "index.md") -> BundleIndex:
|
|
250
|
+
"""Parse a root ``index.md`` into a :class:`BundleIndex`.
|
|
251
|
+
|
|
252
|
+
``body`` is the original Markdown, ``title`` the first H1 when
|
|
253
|
+
present, ``description`` the first non-heading paragraph when present,
|
|
254
|
+
and ``concept_ids`` the normalized internal link targets deduplicated
|
|
255
|
+
in first-seen order.
|
|
256
|
+
|
|
257
|
+
Raises:
|
|
258
|
+
BundleValidationError: If any link target escapes the bundle.
|
|
259
|
+
"""
|
|
260
|
+
title, description = _extract_title_and_description(raw)
|
|
261
|
+
edges = extract_internal_links(raw, source_id="index", source=source)
|
|
262
|
+
concept_ids = list(dict.fromkeys(edge.target_id for edge in edges))
|
|
263
|
+
return BundleIndex(title=title, description=description, body=raw, concept_ids=concept_ids)
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def synthesize_bundle_index(concepts: Sequence[Concept], *, title: str = "Index") -> BundleIndex:
|
|
267
|
+
"""Synthesize an in-memory index for a bundle without a root ``index.md``.
|
|
268
|
+
|
|
269
|
+
Concepts are listed in ascending concept-ID order as bundle-relative
|
|
270
|
+
Markdown links labelled by frontmatter title when available. Nothing
|
|
271
|
+
is written to disk.
|
|
272
|
+
"""
|
|
273
|
+
ordered = sorted(concepts, key=lambda concept: concept.id)
|
|
274
|
+
lines = [f"# {title}", ""]
|
|
275
|
+
lines.extend(
|
|
276
|
+
f"- [{concept.frontmatter.title or concept.id}](/{concept.id}{_MD_SUFFIX})"
|
|
277
|
+
for concept in ordered
|
|
278
|
+
)
|
|
279
|
+
body = "\n".join(lines) + "\n"
|
|
280
|
+
return BundleIndex(
|
|
281
|
+
title=title,
|
|
282
|
+
description=None,
|
|
283
|
+
body=body,
|
|
284
|
+
concept_ids=[concept.id for concept in ordered],
|
|
285
|
+
)
|
okf_agents/bundle.py
ADDED
|
@@ -0,0 +1,288 @@
|
|
|
1
|
+
"""Eager, immutable loading and querying of OKF v0.1 bundles.
|
|
2
|
+
|
|
3
|
+
:class:`OKFBundle` discovers every concept Markdown file under a bundle
|
|
4
|
+
root, parses all of them up front, and builds directed-link indexes for
|
|
5
|
+
lookup, lexical search, and breadth-first traversal. Bundles are
|
|
6
|
+
immutable after loading; every collection-returning method returns a new
|
|
7
|
+
list so callers cannot mutate bundle internals.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import re
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Literal
|
|
15
|
+
|
|
16
|
+
from okf_agents._internal.graph_utils import (
|
|
17
|
+
breadth_first_reachable,
|
|
18
|
+
build_adjacency,
|
|
19
|
+
merge_adjacency,
|
|
20
|
+
)
|
|
21
|
+
from okf_agents._internal.parser import (
|
|
22
|
+
extract_internal_links,
|
|
23
|
+
parse_bundle_index,
|
|
24
|
+
parse_concept,
|
|
25
|
+
synthesize_bundle_index,
|
|
26
|
+
)
|
|
27
|
+
from okf_agents.exceptions import (
|
|
28
|
+
BundleNotFoundError,
|
|
29
|
+
BundleValidationError,
|
|
30
|
+
ConceptNotFoundError,
|
|
31
|
+
)
|
|
32
|
+
from okf_agents.models import BundleIndex, Concept, LinkEdge
|
|
33
|
+
|
|
34
|
+
__all__ = ["OKFBundle"]
|
|
35
|
+
|
|
36
|
+
_RESERVED_FILENAMES = frozenset({"index.md", "log.md"})
|
|
37
|
+
_ROOT_INDEX = "index.md"
|
|
38
|
+
_DIRECTIONS = ("out", "in", "both")
|
|
39
|
+
|
|
40
|
+
# Search field weights per the shared contracts: title > tags > description > body.
|
|
41
|
+
_TITLE_WEIGHT = 4
|
|
42
|
+
_TAGS_WEIGHT = 3
|
|
43
|
+
_DESCRIPTION_WEIGHT = 2
|
|
44
|
+
_BODY_WEIGHT = 1
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class OKFBundle:
|
|
48
|
+
"""A fully loaded OKF bundle: concepts, link graph, and root index.
|
|
49
|
+
|
|
50
|
+
Construct with :meth:`load`; the initializer is an internal detail.
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
def __init__(
|
|
54
|
+
self,
|
|
55
|
+
*,
|
|
56
|
+
root: Path,
|
|
57
|
+
concepts: dict[str, Concept],
|
|
58
|
+
edges_from: dict[str, list[LinkEdge]],
|
|
59
|
+
edges_to: dict[str, list[LinkEdge]],
|
|
60
|
+
index: BundleIndex,
|
|
61
|
+
) -> None:
|
|
62
|
+
self._root = root
|
|
63
|
+
self._concepts = concepts
|
|
64
|
+
self._edges_from = edges_from
|
|
65
|
+
self._edges_to = edges_to
|
|
66
|
+
self._index = index
|
|
67
|
+
all_edges = [edge for edges in edges_from.values() for edge in edges]
|
|
68
|
+
self._out_adjacency, self._in_adjacency = build_adjacency(all_edges)
|
|
69
|
+
self._both_adjacency = merge_adjacency(self._out_adjacency, self._in_adjacency)
|
|
70
|
+
|
|
71
|
+
@classmethod
|
|
72
|
+
def load(cls, path: str | Path) -> OKFBundle:
|
|
73
|
+
"""Eagerly load the bundle rooted at ``path``.
|
|
74
|
+
|
|
75
|
+
Every concept file (any ``.md`` except reserved ``index.md`` and
|
|
76
|
+
``log.md`` at any depth) is parsed up front and validation
|
|
77
|
+
failures are aggregated into one error. A root ``index.md`` is
|
|
78
|
+
parsed when present and synthesized in memory otherwise.
|
|
79
|
+
|
|
80
|
+
Raises:
|
|
81
|
+
BundleNotFoundError: If ``path`` does not exist, is not a
|
|
82
|
+
directory, or cannot be read.
|
|
83
|
+
BundleValidationError: If any concept file is invalid, keyed
|
|
84
|
+
by stable root-relative paths.
|
|
85
|
+
"""
|
|
86
|
+
root = Path(path).resolve()
|
|
87
|
+
if not root.is_dir():
|
|
88
|
+
raise BundleNotFoundError(str(path))
|
|
89
|
+
try:
|
|
90
|
+
concept_files = sorted(
|
|
91
|
+
(
|
|
92
|
+
file
|
|
93
|
+
for file in root.rglob("*.md")
|
|
94
|
+
if file.is_file() and file.name not in _RESERVED_FILENAMES
|
|
95
|
+
),
|
|
96
|
+
key=lambda file: file.relative_to(root).as_posix(),
|
|
97
|
+
)
|
|
98
|
+
except OSError as exc:
|
|
99
|
+
raise BundleNotFoundError(str(path)) from exc
|
|
100
|
+
|
|
101
|
+
concepts: dict[str, Concept] = {}
|
|
102
|
+
failures: dict[str, str] = {}
|
|
103
|
+
for file in concept_files:
|
|
104
|
+
relative = file.relative_to(root).as_posix()
|
|
105
|
+
try:
|
|
106
|
+
raw = file.read_text(encoding="utf-8")
|
|
107
|
+
except UnicodeDecodeError:
|
|
108
|
+
failures[relative] = "file is not valid UTF-8"
|
|
109
|
+
continue
|
|
110
|
+
except OSError as exc:
|
|
111
|
+
failures[relative] = f"file could not be read: {exc}"
|
|
112
|
+
continue
|
|
113
|
+
try:
|
|
114
|
+
concept = parse_concept(raw, bundle_root=root, file_path=file)
|
|
115
|
+
except BundleValidationError as exc:
|
|
116
|
+
failures.update(exc.failed_files)
|
|
117
|
+
continue
|
|
118
|
+
concepts[concept.id] = concept
|
|
119
|
+
|
|
120
|
+
index = cls._load_index(root, concepts, failures)
|
|
121
|
+
if failures:
|
|
122
|
+
raise BundleValidationError(failures)
|
|
123
|
+
assert index is not None # index parsing only fails alongside a failure entry
|
|
124
|
+
|
|
125
|
+
edges_from: dict[str, list[LinkEdge]] = {}
|
|
126
|
+
edges_to: dict[str, list[LinkEdge]] = {}
|
|
127
|
+
for concept_id in sorted(concepts):
|
|
128
|
+
concept = concepts[concept_id]
|
|
129
|
+
source = f"{concept_id}.md"
|
|
130
|
+
edges = [
|
|
131
|
+
edge.model_copy(update={"resolved": edge.target_id in concepts})
|
|
132
|
+
for edge in extract_internal_links(
|
|
133
|
+
concept.body, source_id=concept_id, source=source
|
|
134
|
+
)
|
|
135
|
+
]
|
|
136
|
+
edges_from[concept_id] = edges
|
|
137
|
+
for edge in edges:
|
|
138
|
+
if edge.resolved:
|
|
139
|
+
edges_to.setdefault(edge.target_id, []).append(edge)
|
|
140
|
+
return cls(
|
|
141
|
+
root=root,
|
|
142
|
+
concepts=concepts,
|
|
143
|
+
edges_from=edges_from,
|
|
144
|
+
edges_to=edges_to,
|
|
145
|
+
index=index,
|
|
146
|
+
)
|
|
147
|
+
|
|
148
|
+
@staticmethod
|
|
149
|
+
def _load_index(
|
|
150
|
+
root: Path,
|
|
151
|
+
concepts: dict[str, Concept],
|
|
152
|
+
failures: dict[str, str],
|
|
153
|
+
) -> BundleIndex | None:
|
|
154
|
+
"""Parse the root ``index.md`` or synthesize one in memory."""
|
|
155
|
+
index_path = root / _ROOT_INDEX
|
|
156
|
+
if not index_path.is_file():
|
|
157
|
+
return synthesize_bundle_index(list(concepts.values()))
|
|
158
|
+
try:
|
|
159
|
+
raw = index_path.read_text(encoding="utf-8")
|
|
160
|
+
except (OSError, UnicodeDecodeError) as exc:
|
|
161
|
+
failures[_ROOT_INDEX] = f"index could not be read: {exc}"
|
|
162
|
+
return None
|
|
163
|
+
try:
|
|
164
|
+
return parse_bundle_index(raw, source=_ROOT_INDEX)
|
|
165
|
+
except BundleValidationError as exc:
|
|
166
|
+
failures.update(exc.failed_files)
|
|
167
|
+
return None
|
|
168
|
+
|
|
169
|
+
@property
|
|
170
|
+
def root(self) -> Path:
|
|
171
|
+
"""The resolved bundle root directory."""
|
|
172
|
+
return self._root
|
|
173
|
+
|
|
174
|
+
@property
|
|
175
|
+
def concept_count(self) -> int:
|
|
176
|
+
"""Number of loaded concepts."""
|
|
177
|
+
return len(self._concepts)
|
|
178
|
+
|
|
179
|
+
def get(self, concept_id: str) -> Concept:
|
|
180
|
+
"""Return the concept with the given ID.
|
|
181
|
+
|
|
182
|
+
Raises:
|
|
183
|
+
ConceptNotFoundError: If ``concept_id`` is not in the bundle.
|
|
184
|
+
"""
|
|
185
|
+
try:
|
|
186
|
+
return self._concepts[concept_id]
|
|
187
|
+
except KeyError:
|
|
188
|
+
raise ConceptNotFoundError(concept_id) from None
|
|
189
|
+
|
|
190
|
+
def index(self) -> BundleIndex:
|
|
191
|
+
"""Return a copy of the parsed or synthesized root index."""
|
|
192
|
+
return self._index.model_copy(deep=True)
|
|
193
|
+
|
|
194
|
+
def all_concepts(self) -> list[Concept]:
|
|
195
|
+
"""Return every concept, sorted by concept ID."""
|
|
196
|
+
return [self._concepts[concept_id] for concept_id in sorted(self._concepts)]
|
|
197
|
+
|
|
198
|
+
def search(self, query: str, top_k: int = 5) -> list[Concept]:
|
|
199
|
+
"""Weighted, case-insensitive lexical search over all concepts.
|
|
200
|
+
|
|
201
|
+
Each case-folded query token partially matches (substring) the
|
|
202
|
+
title (weight 4), tags (3), description (2), and body (1); a
|
|
203
|
+
concept must match at least one token. Results rank by descending
|
|
204
|
+
score, then concept ID, capped at ``top_k``.
|
|
205
|
+
|
|
206
|
+
Raises:
|
|
207
|
+
ValueError: If ``top_k`` is less than 1.
|
|
208
|
+
"""
|
|
209
|
+
if top_k < 1:
|
|
210
|
+
raise ValueError(f"top_k must be at least 1, got {top_k}")
|
|
211
|
+
tokens = re.findall(r"\w+", query.casefold())
|
|
212
|
+
if not tokens:
|
|
213
|
+
return []
|
|
214
|
+
scored: list[tuple[int, str]] = []
|
|
215
|
+
for concept_id in sorted(self._concepts):
|
|
216
|
+
concept = self._concepts[concept_id]
|
|
217
|
+
frontmatter = concept.frontmatter
|
|
218
|
+
title = (frontmatter.title or "").casefold()
|
|
219
|
+
tags = [tag.casefold() for tag in frontmatter.tags]
|
|
220
|
+
description = (frontmatter.description or "").casefold()
|
|
221
|
+
body = concept.body.casefold()
|
|
222
|
+
score = 0
|
|
223
|
+
for token in tokens:
|
|
224
|
+
if token in title:
|
|
225
|
+
score += _TITLE_WEIGHT
|
|
226
|
+
if any(token in tag for tag in tags):
|
|
227
|
+
score += _TAGS_WEIGHT
|
|
228
|
+
if token in description:
|
|
229
|
+
score += _DESCRIPTION_WEIGHT
|
|
230
|
+
if token in body:
|
|
231
|
+
score += _BODY_WEIGHT
|
|
232
|
+
if score > 0:
|
|
233
|
+
scored.append((score, concept_id))
|
|
234
|
+
scored.sort(key=lambda item: (-item[0], item[1]))
|
|
235
|
+
return [self._concepts[concept_id] for _, concept_id in scored[:top_k]]
|
|
236
|
+
|
|
237
|
+
def links_from(self, concept_id: str) -> list[LinkEdge]:
|
|
238
|
+
"""Return copies of the outbound edges of a concept in document order.
|
|
239
|
+
|
|
240
|
+
Unresolved edges (broken links) are included with
|
|
241
|
+
``resolved=False``.
|
|
242
|
+
|
|
243
|
+
Raises:
|
|
244
|
+
ConceptNotFoundError: If ``concept_id`` is not in the bundle.
|
|
245
|
+
"""
|
|
246
|
+
self.get(concept_id)
|
|
247
|
+
return [edge.model_copy() for edge in self._edges_from.get(concept_id, [])]
|
|
248
|
+
|
|
249
|
+
def backlinks(self, concept_id: str) -> list[LinkEdge]:
|
|
250
|
+
"""Return copies of the inbound edges of a concept.
|
|
251
|
+
|
|
252
|
+
Edges are ordered by source concept ID, then document order
|
|
253
|
+
within each source.
|
|
254
|
+
|
|
255
|
+
Raises:
|
|
256
|
+
ConceptNotFoundError: If ``concept_id`` is not in the bundle.
|
|
257
|
+
"""
|
|
258
|
+
self.get(concept_id)
|
|
259
|
+
return [edge.model_copy() for edge in self._edges_to.get(concept_id, [])]
|
|
260
|
+
|
|
261
|
+
def neighbors(
|
|
262
|
+
self,
|
|
263
|
+
concept_id: str,
|
|
264
|
+
hops: int = 1,
|
|
265
|
+
direction: Literal["out", "in", "both"] = "out",
|
|
266
|
+
) -> list[Concept]:
|
|
267
|
+
"""Return concepts reachable within ``hops`` links of a concept.
|
|
268
|
+
|
|
269
|
+
Breadth-first over resolved edges only; the starting concept is
|
|
270
|
+
excluded and cycles terminate. Results are ordered by distance,
|
|
271
|
+
then concept ID.
|
|
272
|
+
|
|
273
|
+
Raises:
|
|
274
|
+
ConceptNotFoundError: If ``concept_id`` is not in the bundle.
|
|
275
|
+
ValueError: If ``hops`` is negative or ``direction`` is not
|
|
276
|
+
one of ``"out"``, ``"in"``, or ``"both"``.
|
|
277
|
+
"""
|
|
278
|
+
self.get(concept_id)
|
|
279
|
+
if direction not in _DIRECTIONS:
|
|
280
|
+
raise ValueError(f"direction must be one of {_DIRECTIONS}, got {direction!r}")
|
|
281
|
+
if direction == "out":
|
|
282
|
+
adjacency = self._out_adjacency
|
|
283
|
+
elif direction == "in":
|
|
284
|
+
adjacency = self._in_adjacency
|
|
285
|
+
else:
|
|
286
|
+
adjacency = self._both_adjacency
|
|
287
|
+
reachable = breadth_first_reachable(adjacency, concept_id, hops)
|
|
288
|
+
return [self._concepts[reached_id] for reached_id in reachable]
|