llmwiki2 0.4.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- llmw2/__init__.py +54 -0
- llmw2/__main__.py +8 -0
- llmw2/agents/__init__.py +4 -0
- llmw2/agents/base.py +51 -0
- llmw2/agents/conflicts.py +47 -0
- llmw2/agents/data.py +56 -0
- llmw2/agents/librarian.py +193 -0
- llmw2/agents/linking.py +58 -0
- llmw2/agents/prompts/classifier/consolidate.md +8 -0
- llmw2/agents/prompts/classifier/note_relevance.md +1 -0
- llmw2/agents/prompts/classifier/relate.md +1 -0
- llmw2/agents/prompts/classifier/retrieval_folder.md +1 -0
- llmw2/agents/prompts/classifier/retrieval_note.md +1 -0
- llmw2/agents/prompts/classifier/route.md +11 -0
- llmw2/agents/prompts/classifier/route_open.md +11 -0
- llmw2/agents/prompts/librarian/consolidate.md +17 -0
- llmw2/agents/prompts/librarian/find_match.md +15 -0
- llmw2/agents/prompts/librarian/merge.md +29 -0
- llmw2/agents/prompts/librarian/merge_source.md +26 -0
- llmw2/agents/prompts/librarian/name_folder.md +17 -0
- llmw2/agents/prompts/librarian/relate.md +15 -0
- llmw2/agents/prompts/librarian/route.md +22 -0
- llmw2/agents/prompts/librarian/route_fallback.md +15 -0
- llmw2/agents/prompts/librarian/summarize.md +13 -0
- llmw2/agents/prompts/librarian/summarize_combine.md +13 -0
- llmw2/agents/prompts/librarian/summarize_part.md +17 -0
- llmw2/agents/prompts/librarian/system.md +10 -0
- llmw2/agents/prompts/researcher/answer.md +12 -0
- llmw2/agents/prompts/researcher/navigate.md +25 -0
- llmw2/agents/prompts/researcher/system.md +3 -0
- llmw2/agents/prompts/shared/folder_rules.md +7 -0
- llmw2/agents/prompts/shared/note_rules.md +9 -0
- llmw2/agents/prompts/shared/untrusted.md +1 -0
- llmw2/agents/prompts.py +80 -0
- llmw2/agents/researcher.py +112 -0
- llmw2/agents/revisions.py +293 -0
- llmw2/agents/state.py +232 -0
- llmw2/agents/steps.py +344 -0
- llmw2/agents/steps_classic.py +557 -0
- llmw2/agents/steps_crow.py +402 -0
- llmw2/audit.py +87 -0
- llmw2/batch.py +52 -0
- llmw2/bundle/__init__.py +4 -0
- llmw2/bundle/check.py +118 -0
- llmw2/bundle/document.py +77 -0
- llmw2/bundle/files.py +46 -0
- llmw2/bundle/origin.py +68 -0
- llmw2/bundle/origin_index.py +142 -0
- llmw2/bundle/store.py +578 -0
- llmw2/bundle/tree.py +275 -0
- llmw2/cli.py +266 -0
- llmw2/config.py +349 -0
- llmw2/demo/about-this-demo.md +26 -0
- llmw2/demo/customers/cafe-portello.md +29 -0
- llmw2/demo/customers/index.md +6 -0
- llmw2/demo/customers/refund-policy.md +26 -0
- llmw2/demo/customers/subscription-churn-q2-2026.md +33 -0
- llmw2/demo/customers/wholesale-cafe-onboarding.md +30 -0
- llmw2/demo/ideas/cold-brew-cans.md +28 -0
- llmw2/demo/ideas/index.md +5 -0
- llmw2/demo/ideas/pause-instead-of-cancel.md +26 -0
- llmw2/demo/ideas/roastery-tours.md +27 -0
- llmw2/demo/index.md +16 -0
- llmw2/demo/log.md +35 -0
- llmw2/demo/meetings/board-meeting-june-2026.md +31 -0
- llmw2/demo/meetings/index.md +5 -0
- llmw2/demo/meetings/pricing-meeting-may-2026.md +32 -0
- llmw2/demo/meetings/weekly-sync-10-june-2026.md +26 -0
- llmw2/demo/operations/index.md +4 -0
- llmw2/demo/operations/roasting/aurora-roast-profile.md +33 -0
- llmw2/demo/operations/roasting/index.md +5 -0
- llmw2/demo/operations/roasting/quality-control-and-cupping.md +29 -0
- llmw2/demo/operations/roasting/roaster-maintenance.md +29 -0
- llmw2/demo/operations/shipping/carriers-and-delivery-times.md +26 -0
- llmw2/demo/operations/shipping/index.md +4 -0
- llmw2/demo/operations/shipping/subscription-packing-day.md +30 -0
- llmw2/demo/products/aurora-espresso-blend.md +33 -0
- llmw2/demo/products/ethiopia-guji-seasonal-single-origin.md +32 -0
- llmw2/demo/products/index.md +6 -0
- llmw2/demo/products/night-shift-decaf.md +28 -0
- llmw2/demo/products/price-list-from-july-2026.md +31 -0
- llmw2/demo/raw/20260925-162441-about-this-demo.md +18 -0
- llmw2/demo/raw/20260925-162441-andes-direct-importer.md +16 -0
- llmw2/demo/raw/20260925-162441-aurora-espresso-blend.md +19 -0
- llmw2/demo/raw/20260925-162441-aurora-roast-profile.md +20 -0
- llmw2/demo/raw/20260925-162441-board-meeting-june-2026.md +15 -0
- llmw2/demo/raw/20260925-162441-cafe-portello.md +15 -0
- llmw2/demo/raw/20260925-162441-carriers-and-delivery-times.md +13 -0
- llmw2/demo/raw/20260925-162441-cold-brew-cans.md +15 -0
- llmw2/demo/raw/20260925-162441-ethiopia-guji-seasonal-single-origin.md +16 -0
- llmw2/demo/raw/20260925-162441-green-coffee-price-hedging.md +18 -0
- llmw2/demo/raw/20260925-162441-guji-washing-station-visit-february-2026.md +15 -0
- llmw2/demo/raw/20260925-162441-night-shift-decaf.md +15 -0
- llmw2/demo/raw/20260925-162441-pause-instead-of-cancel.md +13 -0
- llmw2/demo/raw/20260925-162441-price-list-from-july-2026.md +17 -0
- llmw2/demo/raw/20260925-162441-pricing-meeting-may-2026.md +17 -0
- llmw2/demo/raw/20260925-162441-quality-control-and-cupping.md +15 -0
- llmw2/demo/raw/20260925-162441-refund-policy.md +13 -0
- llmw2/demo/raw/20260925-162441-roaster-maintenance.md +15 -0
- llmw2/demo/raw/20260925-162441-roastery-tours.md +15 -0
- llmw2/demo/raw/20260925-162441-subscription-churn-q2-2026.md +19 -0
- llmw2/demo/raw/20260925-162441-subscription-packing-day.md +15 -0
- llmw2/demo/raw/20260925-162441-weekly-sync-10-june-2026.md +12 -0
- llmw2/demo/raw/20260925-162441-wholesale-cafe-onboarding.md +16 -0
- llmw2/demo/sourcing/andes-direct-importer.md +31 -0
- llmw2/demo/sourcing/green-coffee-price-hedging.md +32 -0
- llmw2/demo/sourcing/guji-washing-station-visit-february-2026.md +30 -0
- llmw2/demo/sourcing/index.md +5 -0
- llmw2/errors.py +46 -0
- llmw2/logo.py +65 -0
- llmw2/models/__init__.py +4 -0
- llmw2/models/classifier.py +127 -0
- llmw2/models/client.py +99 -0
- llmw2/models/llm.py +280 -0
- llmw2/models/usage.py +188 -0
- llmw2/py.typed +0 -0
- llmw2/server.py +438 -0
- llmw2/settings.py +356 -0
- llmw2/sources/__init__.py +12 -0
- llmw2/sources/base.py +40 -0
- llmw2/sources/local.py +71 -0
- llmw2/sources/sync.py +102 -0
- llmw2/web/app.js +379 -0
- llmw2/web/graph.js +140 -0
- llmw2/web/img/crow.png +0 -0
- llmw2/web/img/night.jpg +0 -0
- llmw2/web/index.html +100 -0
- llmw2/web/queue.js +139 -0
- llmw2/web/settings.js +288 -0
- llmw2/web/style.css +250 -0
- llmw2/web/util.js +17 -0
- llmw2/web/vendor/d3.min.js +2 -0
- llmw2/web/vendor/fonts/OFL.txt +95 -0
- llmw2/web/vendor/fonts/ibm-plex-sans-latin-400-italic.woff2 +0 -0
- llmw2/web/vendor/fonts/ibm-plex-sans-latin-400-normal.woff2 +0 -0
- llmw2/web/vendor/fonts/ibm-plex-sans-latin-600-normal.woff2 +0 -0
- llmw2/web/vendor/fonts/jetbrains-mono-latin-400-normal.woff2 +0 -0
- llmw2/web/vendor/fonts/jetbrains-mono-latin-500-normal.woff2 +0 -0
- llmw2/web/vendor/fonts/jetbrains-mono-latin-700-normal.woff2 +0 -0
- llmw2/web/vendor/fonts/pixelify-sans-latin-500-normal.woff2 +0 -0
- llmw2/web/vendor/fonts/pixelify-sans-latin-700-normal.woff2 +0 -0
- llmw2/web/vendor/marked.min.js +6 -0
- llmw2/web/vendor/purify.min.js +3 -0
- llmw2/wiki.py +408 -0
- llmwiki2-0.4.1.dist-info/METADATA +405 -0
- llmwiki2-0.4.1.dist-info/RECORD +152 -0
- llmwiki2-0.4.1.dist-info/WHEEL +5 -0
- llmwiki2-0.4.1.dist-info/entry_points.txt +2 -0
- llmwiki2-0.4.1.dist-info/licenses/LICENSE +661 -0
- llmwiki2-0.4.1.dist-info/licenses/LICENSES/Apache-2.0.md +203 -0
- llmwiki2-0.4.1.dist-info/licenses/NOTICE +45 -0
- llmwiki2-0.4.1.dist-info/top_level.txt +1 -0
llmw2/__init__.py
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# Copyright 2026 Federico Cesarini, Marco Sassarini
|
|
2
|
+
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
3
|
+
|
|
4
|
+
"""An LLM wiki that writes Open Knowledge Format bundles.
|
|
5
|
+
|
|
6
|
+
A Librarian files sources into a folder tree of notes, a Researcher answers
|
|
7
|
+
questions from it; in CROW mode a typed classifier takes the decisions and the
|
|
8
|
+
LLM only writes text.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
__version__ = "0.4.1"
|
|
14
|
+
|
|
15
|
+
from llmw2.agents.data import Candidate, NoteDraft, Route, Source
|
|
16
|
+
from llmw2.agents.steps import CLASSIC, CROW, StepContext, Steps
|
|
17
|
+
from llmw2.audit import AUDIT_MIN, FolderHealth, RouteProposal
|
|
18
|
+
from llmw2.batch import BatchReport, Item
|
|
19
|
+
from llmw2.bundle.origin import Origin
|
|
20
|
+
from llmw2.config import WikiConfig
|
|
21
|
+
from llmw2.errors import ConfigError, InputError, ModelError, StepError, WikiError
|
|
22
|
+
from llmw2.sources import Change, ChangeBatch, LocalFolderSource, SourceConnector, SyncReport, sync
|
|
23
|
+
from llmw2.wiki import Wiki
|
|
24
|
+
|
|
25
|
+
__all__ = [
|
|
26
|
+
"AUDIT_MIN",
|
|
27
|
+
"CLASSIC",
|
|
28
|
+
"CROW",
|
|
29
|
+
"BatchReport",
|
|
30
|
+
"Candidate",
|
|
31
|
+
"Change",
|
|
32
|
+
"ChangeBatch",
|
|
33
|
+
"ConfigError",
|
|
34
|
+
"FolderHealth",
|
|
35
|
+
"InputError",
|
|
36
|
+
"Item",
|
|
37
|
+
"LocalFolderSource",
|
|
38
|
+
"ModelError",
|
|
39
|
+
"NoteDraft",
|
|
40
|
+
"Origin",
|
|
41
|
+
"Route",
|
|
42
|
+
"RouteProposal",
|
|
43
|
+
"Source",
|
|
44
|
+
"SourceConnector",
|
|
45
|
+
"StepContext",
|
|
46
|
+
"StepError",
|
|
47
|
+
"Steps",
|
|
48
|
+
"SyncReport",
|
|
49
|
+
"Wiki",
|
|
50
|
+
"WikiConfig",
|
|
51
|
+
"WikiError",
|
|
52
|
+
"__version__",
|
|
53
|
+
"sync",
|
|
54
|
+
]
|
llmw2/__main__.py
ADDED
llmw2/agents/__init__.py
ADDED
llmw2/agents/base.py
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# Copyright 2026 Federico Cesarini, Marco Sassarini
|
|
2
|
+
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
3
|
+
|
|
4
|
+
"""What the steps share: the trace of decisions and the helpers every prompt and flow uses."""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
from collections.abc import Iterable
|
|
9
|
+
from dataclasses import dataclass, field
|
|
10
|
+
from typing import Literal, TypeVar
|
|
11
|
+
|
|
12
|
+
from llmw2.bundle.tree import Folder, Note, slugify
|
|
13
|
+
|
|
14
|
+
T = TypeVar("T")
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@dataclass
|
|
18
|
+
class Decision:
|
|
19
|
+
"""One decision of an ingest or a question, kept to calibrate thresholds (CROW §5.3)."""
|
|
20
|
+
|
|
21
|
+
step: str # route, match, consolidate, relate, navigate, select, …
|
|
22
|
+
decider: Literal["llm", "classifier", "rule", "code"] # "code": a check the code makes alone (revisions)
|
|
23
|
+
choice: str
|
|
24
|
+
confidence: float | None = None
|
|
25
|
+
fallback: bool = False # the classifier was not confident (or failed) and the LLM decided
|
|
26
|
+
scores: dict[str, float] = field(default_factory=dict) # classifier probability per option / candidate
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def list_folders(folders: list[Folder]) -> str:
|
|
30
|
+
"""`- name: description` lines, the way every prompt shows folders."""
|
|
31
|
+
return "\n".join(f"- {f.name}: {f.description or '(no description)'}" for f in folders) or "(none)"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def list_notes(notes: list[Note]) -> str:
|
|
35
|
+
"""`- path: Title — summary` lines, the way every prompt shows notes."""
|
|
36
|
+
return "\n".join(f"- {n.rel}: {n.title} — {n.summary}" for n in notes) or "(none)"
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def unique_objects(items: Iterable[T]) -> list[T]:
|
|
40
|
+
"""Keep the first occurrence of each object, in order."""
|
|
41
|
+
seen: set[int] = set()
|
|
42
|
+
out = []
|
|
43
|
+
for item in items:
|
|
44
|
+
if id(item) not in seen:
|
|
45
|
+
seen.add(id(item))
|
|
46
|
+
out.append(item)
|
|
47
|
+
return out
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def clean_tags(tags: list[str]) -> list[str]:
|
|
51
|
+
return list(dict.fromkeys(t for t in (slugify(t, max_len=30, fallback="") for t in tags) if t))[:6]
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# Copyright 2026 Federico Cesarini, Marco Sassarini
|
|
2
|
+
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
3
|
+
|
|
4
|
+
"""When the notes read to answer a question disagree on its answer, the answer must report every value (CROW).
|
|
5
|
+
|
|
6
|
+
One typed question reads the passage of each note that is closest to the question; if it says the notes disagree,
|
|
7
|
+
the answer step gets a hint added to the question.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import re
|
|
13
|
+
|
|
14
|
+
from llmw2.bundle.tree import Note
|
|
15
|
+
|
|
16
|
+
__all__ = ["CONFLICT", "CONFLICT_HINT", "CONFLICT_MIN", "MAX_NOTES", "PASSAGE_CHARS", "conflict_state"]
|
|
17
|
+
|
|
18
|
+
CONFLICT_MIN = 0.4 # the probability that the notes disagree, above which the hint is added
|
|
19
|
+
PASSAGE_CHARS = 900
|
|
20
|
+
MAX_NOTES = 8
|
|
21
|
+
|
|
22
|
+
CONFLICT = "The notes do not all agree on the answer to the question: at least one gives a value that another contradicts."
|
|
23
|
+
CONFLICT_HINT = ( # added to the question of the answer step, after the history hint if there is one
|
|
24
|
+
"(The notes may disagree on this. If two notes give different values for what the question asks, report every value, "
|
|
25
|
+
"say which note gives which and which source is more recent, and do not pick one silently.)"
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
_WORD = re.compile(r"\w{3,}")
|
|
29
|
+
_BLANK_LINE = re.compile(r"\n\s*\n")
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _words(text: str) -> set[str]:
|
|
33
|
+
return set(_WORD.findall(text.lower()))
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _passage(question: str, note: Note) -> str:
|
|
37
|
+
"""The paragraph of the note that shares most words with the question (the first on a tie), cut."""
|
|
38
|
+
paragraphs = [p.strip() for p in _BLANK_LINE.split(note.main_body()) if p.strip()]
|
|
39
|
+
asked = _words(question)
|
|
40
|
+
best = max(paragraphs, key=lambda p: len(asked & _words(p)), default="") # max keeps the first of equals
|
|
41
|
+
return best[:PASSAGE_CHARS]
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def conflict_state(question: str, notes: list[Note]) -> str:
|
|
45
|
+
"""The question and, for each of the first `MAX_NOTES` notes, its title and the passage closest to the question."""
|
|
46
|
+
blocks = [f"### Note: {n.title}\n{_passage(question, n)}" for n in notes[:MAX_NOTES]]
|
|
47
|
+
return f"Question: {question}\n\n" + "\n\n".join(blocks)
|
llmw2/agents/data.py
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# Copyright 2026 Federico Cesarini, Marco Sassarini
|
|
2
|
+
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
3
|
+
|
|
4
|
+
"""The data the steps pass to one another: a source, the candidate note, a route, a rewritten note."""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
from dataclasses import dataclass, field
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from pydantic import BaseModel
|
|
12
|
+
|
|
13
|
+
from llmw2.bundle.origin import Origin
|
|
14
|
+
from llmw2.bundle.tree import Folder
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@dataclass
|
|
18
|
+
class Source:
|
|
19
|
+
text: str
|
|
20
|
+
title: str | None = None
|
|
21
|
+
resource: str | None = None # URL or path of the original, if any
|
|
22
|
+
origin: Origin | None = None # where the original lives at its source, if it has one
|
|
23
|
+
fields: dict[str, Any] = field(default_factory=dict) # user data, uninterpreted by the core
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass
|
|
27
|
+
class Candidate:
|
|
28
|
+
"""The note an ingest would write (CROW step 0)."""
|
|
29
|
+
|
|
30
|
+
title: str
|
|
31
|
+
summary: str
|
|
32
|
+
tags: list[str]
|
|
33
|
+
body: str
|
|
34
|
+
fields: dict[str, Any] = field(default_factory=dict) # written to the note's frontmatter under `fields:`
|
|
35
|
+
state: str = "" # what the routing decisions read; empty: built from the source text (see `agents.state`)
|
|
36
|
+
|
|
37
|
+
def compact(self, max_chars: int) -> str:
|
|
38
|
+
"""Title, summary and opening passage: the state every decision reads."""
|
|
39
|
+
return f"Title: {self.title}\nSummary: {self.summary}\n\n{self.body}".strip()[:max_chars]
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@dataclass
|
|
43
|
+
class Route:
|
|
44
|
+
folder: Folder # where the note goes
|
|
45
|
+
new: bool # file it in a new subfolder of `folder`
|
|
46
|
+
candidates: list[Folder] # folders whose notes are checked for consolidation
|
|
47
|
+
# (name, description) per level to open below `folder`, reusing a subfolder of the same name; the note goes in the last one
|
|
48
|
+
create: list[tuple[str, str]] = field(default_factory=list)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class NoteDraft(BaseModel):
|
|
52
|
+
title: str
|
|
53
|
+
summary: str
|
|
54
|
+
tags: list[str] = []
|
|
55
|
+
body: str
|
|
56
|
+
fields: dict[str, Any] = {}
|
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
# Copyright 2026 Federico Cesarini, Marco Sassarini
|
|
2
|
+
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
3
|
+
|
|
4
|
+
"""The Librarian files a source into the wiki.
|
|
5
|
+
|
|
6
|
+
summarize → route → match → consolidate → merge | (name_folder →) write → relate → link
|
|
7
|
+
|
|
8
|
+
The flow is fixed; each step is a function of the `Steps` it is given (classic, CROW or custom), and the
|
|
9
|
+
output of every step is checked before it is used.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
from dataclasses import asdict, dataclass, field
|
|
15
|
+
from functools import cached_property
|
|
16
|
+
from typing import Any, Literal
|
|
17
|
+
|
|
18
|
+
from llmw2.agents import linking, revisions
|
|
19
|
+
from llmw2.agents.base import Decision, clean_tags, unique_objects
|
|
20
|
+
from llmw2.agents.data import Candidate, NoteDraft, Route, Source
|
|
21
|
+
from llmw2.agents.prompts import Prompts
|
|
22
|
+
from llmw2.agents.steps import (
|
|
23
|
+
StepContext,
|
|
24
|
+
Steps,
|
|
25
|
+
check_consolidate,
|
|
26
|
+
check_match,
|
|
27
|
+
check_merge,
|
|
28
|
+
check_name_folder,
|
|
29
|
+
check_relate,
|
|
30
|
+
check_route,
|
|
31
|
+
check_summarize,
|
|
32
|
+
decision_hash,
|
|
33
|
+
)
|
|
34
|
+
from llmw2.bundle.origin import content_hash
|
|
35
|
+
from llmw2.bundle.store import WikiStore
|
|
36
|
+
from llmw2.config import WikiConfig
|
|
37
|
+
from llmw2.models.classifier import Classifier
|
|
38
|
+
from llmw2.models.llm import LLM
|
|
39
|
+
from llmw2.models.usage import UsageReport
|
|
40
|
+
|
|
41
|
+
__all__ = ["Candidate", "IngestResult", "Librarian", "NoteDraft", "Prepared", "Route", "Source"]
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@dataclass
|
|
45
|
+
class IngestResult:
|
|
46
|
+
action: Literal["created", "merged", "unchanged"] # "unchanged": nothing was written
|
|
47
|
+
note: str # bundle path of the note written ("" when unchanged and no note cites the source)
|
|
48
|
+
title: str
|
|
49
|
+
folder: str # "/" or "/finance/pricing"
|
|
50
|
+
created_folders: list[str]
|
|
51
|
+
related: list[str]
|
|
52
|
+
decisions: list[Decision]
|
|
53
|
+
usage: UsageReport = field(default_factory=UsageReport)
|
|
54
|
+
|
|
55
|
+
def to_dict(self) -> dict[str, Any]:
|
|
56
|
+
return {**asdict(self), "usage": self.usage.to_dict()}
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@dataclass
|
|
60
|
+
class Prepared:
|
|
61
|
+
"""A source summarized and waiting to be filed: what `Librarian.prepare` hands to `Librarian.file`."""
|
|
62
|
+
|
|
63
|
+
source: Source
|
|
64
|
+
candidate: Candidate
|
|
65
|
+
usage: UsageReport # the model calls made so far for this source
|
|
66
|
+
decisions: list[Decision] = field(default_factory=list)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class Librarian:
|
|
70
|
+
"""One instance serves one ingest, so its trace of decisions is never shared."""
|
|
71
|
+
|
|
72
|
+
def __init__(
|
|
73
|
+
self, store: WikiStore, llm: LLM, prompts: Prompts, cfg: WikiConfig, steps: Steps, classifier: Classifier | None = None
|
|
74
|
+
) -> None:
|
|
75
|
+
self.store = store
|
|
76
|
+
self.cfg = cfg
|
|
77
|
+
self.steps = steps
|
|
78
|
+
self.ctx = StepContext(llm, classifier, prompts, cfg, system_prompt="librarian/system")
|
|
79
|
+
|
|
80
|
+
@cached_property
|
|
81
|
+
def hash(self) -> str:
|
|
82
|
+
"""What the provenance says decided: the steps, prompts and decision parameters (see `decision_hash`)."""
|
|
83
|
+
return decision_hash(self.steps, self.cfg, self.ctx.prompts)
|
|
84
|
+
|
|
85
|
+
def ingest(self, source: Source) -> IngestResult:
|
|
86
|
+
return self.file(self.prepare(source))
|
|
87
|
+
|
|
88
|
+
def prepare(self, source: Source) -> Prepared:
|
|
89
|
+
"""The part of an ingest that does not read the tree: summarize. Needs no lock, and may run in parallel."""
|
|
90
|
+
self.ctx.source = source
|
|
91
|
+
with self.ctx.llm.usage.span() as used:
|
|
92
|
+
cand = check_summarize(self.steps.summarize(self.ctx, source))
|
|
93
|
+
return Prepared(source, cand, used, list(self.ctx.decisions))
|
|
94
|
+
|
|
95
|
+
def file(self, prepared: Prepared) -> IngestResult:
|
|
96
|
+
"""The rest: from a fresh `load()` to the log. The caller holds the wiki's lock."""
|
|
97
|
+
steps, ctx = self.steps, self.ctx
|
|
98
|
+
source, cand = prepared.source, prepared.candidate
|
|
99
|
+
ctx.source = source
|
|
100
|
+
ctx.candidate = cand
|
|
101
|
+
ctx.decisions = list(prepared.decisions)
|
|
102
|
+
with ctx.llm.usage.span() as used:
|
|
103
|
+
root = self.store.load()
|
|
104
|
+
route = check_route(steps.route(ctx, root, cand), root, self.cfg)
|
|
105
|
+
pool = unique_objects(n for f in [*route.candidates, route.folder] for n in f.notes)
|
|
106
|
+
match = check_match(steps.match(ctx, pool, cand), pool)
|
|
107
|
+
created: list[str] = []
|
|
108
|
+
merging = match is not None and check_consolidate(steps.consolidate(ctx, match, cand))
|
|
109
|
+
revised, archived = False, ""
|
|
110
|
+
if match is not None and merging:
|
|
111
|
+
draft = revisions.revise(ctx, match, source) if self.cfg.revisions == "replace" else None
|
|
112
|
+
if draft is not None:
|
|
113
|
+
archived = self.store.archive_note(match)
|
|
114
|
+
revised = True
|
|
115
|
+
else:
|
|
116
|
+
draft = check_merge(steps.merge(ctx, match, source))
|
|
117
|
+
raw, ref = self.save_source(source, cand)
|
|
118
|
+
note = self.store.update_note(
|
|
119
|
+
match,
|
|
120
|
+
title=draft.title,
|
|
121
|
+
summary=draft.summary,
|
|
122
|
+
body=draft.body,
|
|
123
|
+
tags=clean_tags(draft.tags),
|
|
124
|
+
source=ref,
|
|
125
|
+
fields=draft.fields,
|
|
126
|
+
previous_version=archived or None,
|
|
127
|
+
)
|
|
128
|
+
action: Literal["created", "merged"] = "merged"
|
|
129
|
+
else:
|
|
130
|
+
folder = route.folder
|
|
131
|
+
if route.create:
|
|
132
|
+
for name, description in route.create:
|
|
133
|
+
folder, is_new = self.store.create_folder(folder, name, description)
|
|
134
|
+
created += [folder.label] if is_new else []
|
|
135
|
+
elif route.new and ctx.can_create(folder):
|
|
136
|
+
name, description = check_name_folder(steps.name_folder(ctx, folder, cand))
|
|
137
|
+
folder, is_new = self.store.create_folder(folder, name, description)
|
|
138
|
+
created += [folder.label] if is_new else []
|
|
139
|
+
raw, ref = self.save_source(source, cand)
|
|
140
|
+
note = self.store.write_note(
|
|
141
|
+
folder,
|
|
142
|
+
title=cand.title,
|
|
143
|
+
summary=cand.summary,
|
|
144
|
+
body=cand.body,
|
|
145
|
+
tags=clean_tags(cand.tags),
|
|
146
|
+
source=ref,
|
|
147
|
+
fields=cand.fields,
|
|
148
|
+
)
|
|
149
|
+
action = "created"
|
|
150
|
+
others = [n for n in unique_objects([*pool, *note.folder.notes]) if n is not note] if note.folder else pool
|
|
151
|
+
if ctx.classifier is not None and self.cfg.relate_rare_values:
|
|
152
|
+
found = linking.rare_scores(root, note, {n.rel for n in others} | {note.rel})
|
|
153
|
+
others = [*others, *(n for n, _ in found)]
|
|
154
|
+
if found:
|
|
155
|
+
ctx.decide("relate_candidates", "code", f"{len(found)} by rare values", scores={n.rel: round(s, 3) for n, s in found})
|
|
156
|
+
related = check_relate(steps.relate(ctx, note, others), note, others) if others else []
|
|
157
|
+
related = related[: self.cfg.max_links]
|
|
158
|
+
for other in related:
|
|
159
|
+
self.store.link(note, other)
|
|
160
|
+
verb = ("Revised" if revised else "Merged") if action == "merged" else "Created"
|
|
161
|
+
via = f", via {','.join(self.cfg.llm.pin_provider)}" if self.cfg.llm.pin_provider else ""
|
|
162
|
+
kept = f", previous version archived at {archived}" if revised else ""
|
|
163
|
+
# CROW: the classifier did not settle the route and the LLM chose the folder (read by `Wiki.taxonomy_health`)
|
|
164
|
+
llm_decides = self.cfg.mode == "crow" # in classic the LLM routes every note: that is no fallback
|
|
165
|
+
fallback = any(d.step == "route" and (d.fallback or (llm_decides and d.decider == "llm")) for d in ctx.decisions)
|
|
166
|
+
routed = ", routed by fallback" if fallback else ""
|
|
167
|
+
text = f"from {raw} ({steps.name} {self.hash}, {self.cfg.llm.model}{via}){kept}{routed}"
|
|
168
|
+
self.store.append_log(verb, text, note.rel, note.title)
|
|
169
|
+
total = prepared.usage.copy()
|
|
170
|
+
total.merge(used)
|
|
171
|
+
return IngestResult(
|
|
172
|
+
action,
|
|
173
|
+
note.rel,
|
|
174
|
+
note.title,
|
|
175
|
+
note.folder.label if note.folder else "/",
|
|
176
|
+
created,
|
|
177
|
+
[r.rel for r in related],
|
|
178
|
+
ctx.decisions,
|
|
179
|
+
total,
|
|
180
|
+
)
|
|
181
|
+
|
|
182
|
+
def save_source(self, source: Source, cand: Candidate) -> tuple[str, dict[str, Any]]:
|
|
183
|
+
"""Keep the raw copy and build the note's source entry; called right before the note is written.
|
|
184
|
+
|
|
185
|
+
Saved this late so an interrupted ingest leaves no raw copy that no note cites.
|
|
186
|
+
"""
|
|
187
|
+
title = source.title or cand.title
|
|
188
|
+
raw = self.store.save_raw(source.text, title=title, resource=source.resource, origin=source.origin)
|
|
189
|
+
ref: dict[str, Any] = {"resource": raw, "title": title, "hash": content_hash(source.text)}
|
|
190
|
+
if source.origin is not None:
|
|
191
|
+
ref["origin"] = source.origin.to_dict()
|
|
192
|
+
ref["steps"] = {"name": self.steps.name, "hash": self.hash}
|
|
193
|
+
return raw, ref
|
llmw2/agents/linking.py
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"""See-also candidates found by code: notes anywhere in the wiki that share rare values with a note.
|
|
2
|
+
|
|
3
|
+
The route only visits some folders, so two notes with the same facts in distant folders would never meet in `relate`.
|
|
4
|
+
Values held by few notes (codes, amounts, dates, names) bring them together. No model call.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import math
|
|
10
|
+
import re
|
|
11
|
+
|
|
12
|
+
from llmw2.agents.steps_classic import incoming_values
|
|
13
|
+
from llmw2.bundle.store import body_hash
|
|
14
|
+
from llmw2.bundle.tree import Folder, Note, split_see_also
|
|
15
|
+
|
|
16
|
+
RARE_TOP = 40 # most notes added to the candidates of `relate`
|
|
17
|
+
RARE_MAX_SHARE = 0.05 # a value is rare when at most this share of the notes hold it...
|
|
18
|
+
RARE_MIN_CAP = 3 # ...or this many, in a small wiki
|
|
19
|
+
INDEGREE_CAP = 12 # a note with this many See-also entries is not offered
|
|
20
|
+
_CACHE_MAX = 50_000
|
|
21
|
+
_YEAR = re.compile(r"^(19|20)\d\d$")
|
|
22
|
+
|
|
23
|
+
_cache: dict[tuple[str, str, str, str], frozenset[str]] = {}
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def note_values(note: Note) -> frozenset[str]:
|
|
27
|
+
"""The values of a note (`incoming_values` of its text), cached by what they are computed from."""
|
|
28
|
+
key = (note.rel, body_hash(note.main_body()), note.title, note.summary)
|
|
29
|
+
values = _cache.get(key)
|
|
30
|
+
if values is None:
|
|
31
|
+
if len(_cache) > _CACHE_MAX:
|
|
32
|
+
_cache.clear()
|
|
33
|
+
values = _cache[key] = frozenset(incoming_values(note.full_text()))
|
|
34
|
+
return values
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def rare_scores(root: Folder, note: Note, exclude: set[str]) -> list[tuple[Note, float]]:
|
|
38
|
+
"""The notes that share rare values with `note`, with their score: best first, at most `RARE_TOP`."""
|
|
39
|
+
notes = root.all_notes()
|
|
40
|
+
total = len(notes)
|
|
41
|
+
held = {n.rel: note_values(n) for n in notes}
|
|
42
|
+
df: dict[str, int] = {}
|
|
43
|
+
for values in held.values():
|
|
44
|
+
for v in values:
|
|
45
|
+
df[v] = df.get(v, 0) + 1
|
|
46
|
+
cap = max(RARE_MIN_CAP, RARE_MAX_SHARE * total)
|
|
47
|
+
weight = {v: math.log(total / c) for v, c in df.items() if c <= cap and not _YEAR.match(v)}
|
|
48
|
+
mine = {v for v in note_values(note) if v in weight}
|
|
49
|
+
scored: list[tuple[Note, float]] = []
|
|
50
|
+
for other in notes:
|
|
51
|
+
if other.rel == note.rel or other.rel in exclude:
|
|
52
|
+
continue
|
|
53
|
+
score = sum(weight[v] for v in held[other.rel] & mine)
|
|
54
|
+
if score <= 0 or len(split_see_also(other.body)[1]) >= INDEGREE_CAP:
|
|
55
|
+
continue
|
|
56
|
+
scored.append((other, score))
|
|
57
|
+
scored.sort(key=lambda item: (-item[1], item[0].rel))
|
|
58
|
+
return scored[:RARE_TOP]
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
## instructions
|
|
2
|
+
The state holds an incoming note followed by an existing note. Should the incoming content modify the existing note, or become a new note?
|
|
3
|
+
|
|
4
|
+
## Modify
|
|
5
|
+
The incoming content is about the same subject as the existing note and adds, updates or corrects its information.
|
|
6
|
+
|
|
7
|
+
## New note
|
|
8
|
+
The incoming content is about a related but distinct subject that deserves a note of its own.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
Does the incoming content concern the same subject as the note titled "{{title}}", summarized as "{{summary}}"?
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
Would a reader of this note want to open the note titled "{{title}}", summarized as "{{summary}}", because it covers a closely connected subject?
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
Is the folder "{{name}}", described as "{{description}}", likely to contain notes that help answer the question?
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
Does the note titled "{{title}}", summarized as "{{summary}}", help answer the question?
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
## instructions
|
|
2
|
+
Where does the incoming note belong, relative to the wiki folder {{folder}}? Pick the subfolder whose description covers the subject of the note, or one of the placement options.
|
|
3
|
+
|
|
4
|
+
## Here
|
|
5
|
+
The description of the folder {{folder}} covers the subject of the note, and no listed subfolder covers it better.
|
|
6
|
+
|
|
7
|
+
## New subfolder
|
|
8
|
+
The subject of the note belongs under the folder {{folder}}, but neither the folder's description nor any listed subfolder covers it.
|
|
9
|
+
|
|
10
|
+
## None of these
|
|
11
|
+
The subject of the note belongs to a different part of the wiki than the folder {{folder}}.
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
## instructions
|
|
2
|
+
Where does the incoming note belong, relative to the wiki folder {{folder}}? Pick the subfolder whose description covers the subject of the note, or one of the placement options.
|
|
3
|
+
|
|
4
|
+
## Here
|
|
5
|
+
The subject of the note matches the folder {{folder}} as a whole, more than any single subfolder.
|
|
6
|
+
|
|
7
|
+
## New subfolder
|
|
8
|
+
The subject of the note fits under the folder {{folder}} and needs a subfolder of its own, distinct from the listed subfolders.
|
|
9
|
+
|
|
10
|
+
## None of these
|
|
11
|
+
The subject of the note belongs to a different part of the wiki than the folder {{folder}}.
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
The incoming note may be about the same subject as the existing note. Decide what to do with it.
|
|
2
|
+
|
|
3
|
+
- "modify": the incoming content updates, corrects or extends the subject of the existing note, so it is merged into it.
|
|
4
|
+
- "new": the incoming content is about a related but distinct subject, so it gets its own note next to the existing one.
|
|
5
|
+
|
|
6
|
+
When unsure, answer "new": a duplicate can be merged later, while a wrong merge blurs two subjects into one note.
|
|
7
|
+
|
|
8
|
+
<note>
|
|
9
|
+
{{note}}
|
|
10
|
+
</note>
|
|
11
|
+
|
|
12
|
+
<existing>
|
|
13
|
+
{{existing}}
|
|
14
|
+
</existing>
|
|
15
|
+
|
|
16
|
+
Reply with JSON:
|
|
17
|
+
{"reasoning": "...", "decision": "modify" | "new"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
Is the incoming note about the same subject as one of the existing notes below?
|
|
2
|
+
|
|
3
|
+
Same subject means a reader would expect both texts in one note: the same entity, concept, process, event or decision, possibly with new, updated or corrected information. Notes that are only related (same area, different subject) do not match.
|
|
4
|
+
|
|
5
|
+
<note>
|
|
6
|
+
{{note}}
|
|
7
|
+
</note>
|
|
8
|
+
|
|
9
|
+
Existing notes (path: title — summary):
|
|
10
|
+
<notes>
|
|
11
|
+
{{notes}}
|
|
12
|
+
</notes>
|
|
13
|
+
|
|
14
|
+
Reply with JSON; "match" is the exact path of the matching note, or null when none matches or you are unsure:
|
|
15
|
+
{"reasoning": "...", "match": "<path>" | null}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
Merge an incoming note into an existing wiki note and return the complete updated note.
|
|
2
|
+
|
|
3
|
+
{{shared/note_rules}}
|
|
4
|
+
|
|
5
|
+
How to merge:
|
|
6
|
+
- The result is the existing note, updated: keep its title, summary, headings and order of sections, and its wording where nothing changed. Do not rewrite or reorganise it around the incoming note.
|
|
7
|
+
- Keep every fact of the existing note that the incoming note does not contradict.
|
|
8
|
+
- Put each new fact in the section where it belongs; add a section only when none fits.
|
|
9
|
+
- When the incoming note updates or contradicts the existing one, keep the newer fact and state the change in place, with the date of the incoming note ("As of March 2026 the limit is 40, previously 30"), unless the incoming note is clearly older than the existing one.
|
|
10
|
+
- Change the title and summary only if the subject itself changed, not because the incoming note is longer or has another focus.
|
|
11
|
+
|
|
12
|
+
Existing note
|
|
13
|
+
Title: {{title}}
|
|
14
|
+
Summary: {{summary}}
|
|
15
|
+
Tags: {{tags}}
|
|
16
|
+
<existing>
|
|
17
|
+
{{body}}
|
|
18
|
+
</existing>
|
|
19
|
+
|
|
20
|
+
Incoming note ({{incoming_from}})
|
|
21
|
+
Title: {{incoming_title}}
|
|
22
|
+
Summary: {{incoming_summary}}
|
|
23
|
+
Tags: {{incoming_tags}}
|
|
24
|
+
<note>
|
|
25
|
+
{{incoming_body}}
|
|
26
|
+
</note>
|
|
27
|
+
|
|
28
|
+
Reply with JSON:
|
|
29
|
+
{"title": "...", "summary": "...", "tags": ["..."], "body": "..."}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
Merge a new source into an existing wiki note and return the complete updated note.
|
|
2
|
+
|
|
3
|
+
{{shared/note_rules}}
|
|
4
|
+
|
|
5
|
+
How to merge:
|
|
6
|
+
- Keep every fact of the existing note that the source does not contradict.
|
|
7
|
+
- Put each new fact in the section where it belongs; add a section only when none fits.
|
|
8
|
+
- When the source updates or contradicts the note, keep the newer fact and state the change in place ("As of March 2026 the limit is 40, previously 30"), unless the source is clearly older than the note.
|
|
9
|
+
- Never shorten the note to the size of the source and never drop sections.
|
|
10
|
+
- Change the title and summary only if the scope of the subject changed.
|
|
11
|
+
|
|
12
|
+
Existing note
|
|
13
|
+
Title: {{title}}
|
|
14
|
+
Summary: {{summary}}
|
|
15
|
+
Tags: {{tags}}
|
|
16
|
+
<existing>
|
|
17
|
+
{{body}}
|
|
18
|
+
</existing>
|
|
19
|
+
|
|
20
|
+
New source
|
|
21
|
+
<source>
|
|
22
|
+
{{source_text}}
|
|
23
|
+
</source>
|
|
24
|
+
|
|
25
|
+
Reply with JSON:
|
|
26
|
+
{"title": "...", "summary": "...", "tags": ["..."], "body": "..."}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
A new subfolder is being created under {{parent}} ({{parent_description}}) to hold the incoming note. Name it and describe it.
|
|
2
|
+
|
|
3
|
+
Existing subfolders of {{parent}} (name: description):
|
|
4
|
+
{{siblings}}
|
|
5
|
+
|
|
6
|
+
<note>
|
|
7
|
+
{{note}}
|
|
8
|
+
</note>
|
|
9
|
+
|
|
10
|
+
{{shared/folder_rules}}
|
|
11
|
+
|
|
12
|
+
The name: 1 to 3 lowercase words joined by hyphens, letters a-z and digits only; a plural noun for collections ("suppliers", "incident-reports"); in the language of the existing folders, English if there are none. It must not duplicate or overlap an existing subfolder.
|
|
13
|
+
Directly under the root ("/"), name the broad area the note belongs to ("engineering", "finance", "customers"), never its specific topic: specific topics become subfolders of that area later.
|
|
14
|
+
The description: one line of at most 150 characters stating what belongs in the folder — the category, not this single note.
|
|
15
|
+
|
|
16
|
+
Reply with JSON:
|
|
17
|
+
{"name": "...", "description": "..."}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
Choose the existing notes that deserve a "See also" link from the note below.
|
|
2
|
+
|
|
3
|
+
Link a note only when a reader of this note would want to open it next: it covers the same entity from another angle, a part or the whole of it, a cause, a consequence, a prerequisite or a direct alternative. Do not link notes that only share a broad area. At most {{max_links}} links; an empty list is the right answer when nothing qualifies.
|
|
4
|
+
|
|
5
|
+
<note>
|
|
6
|
+
{{note}}
|
|
7
|
+
</note>
|
|
8
|
+
|
|
9
|
+
Candidates (path: title — summary):
|
|
10
|
+
<notes>
|
|
11
|
+
{{notes}}
|
|
12
|
+
</notes>
|
|
13
|
+
|
|
14
|
+
Reply with JSON; use exact paths from the list, best first:
|
|
15
|
+
{"reasoning": "...", "related": ["<path>", "..."]}
|