sessionwiki 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sessionwiki/__init__.py +74 -0
- sessionwiki/__main__.py +6 -0
- sessionwiki/_frontmatter.py +118 -0
- sessionwiki/_system_pages.py +44 -0
- sessionwiki/adapter_config.py +124 -0
- sessionwiki/adapters/__init__.py +134 -0
- sessionwiki/adapters/base.py +191 -0
- sessionwiki/adapters/claude_code.py +77 -0
- sessionwiki/adapters/codex_cli.py +226 -0
- sessionwiki/adapters/contrib/__init__.py +1 -0
- sessionwiki/adapters/contrib/chatgpt.py +252 -0
- sessionwiki/adapters/contrib/copilot_chat.py +121 -0
- sessionwiki/adapters/contrib/copilot_cli.py +115 -0
- sessionwiki/adapters/contrib/cursor.py +100 -0
- sessionwiki/adapters/contrib/gemini_cli.py +74 -0
- sessionwiki/adapters/contrib/obsidian.py +121 -0
- sessionwiki/adapters/opencode.py +169 -0
- sessionwiki/adapters/status.py +58 -0
- sessionwiki/adapters/zcode.py +45 -0
- sessionwiki/audit.py +174 -0
- sessionwiki/backlinks.py +224 -0
- sessionwiki/build.py +2786 -0
- sessionwiki/cache.py +401 -0
- sessionwiki/candidates.py +309 -0
- sessionwiki/categories.py +158 -0
- sessionwiki/changelog_timeline.py +422 -0
- sessionwiki/cli.py +1687 -0
- sessionwiki/compare.py +404 -0
- sessionwiki/completion.py +191 -0
- sessionwiki/confidence.py +179 -0
- sessionwiki/config.py +74 -0
- sessionwiki/config_schedule.py +77 -0
- sessionwiki/context_md.py +182 -0
- sessionwiki/convert.py +1746 -0
- sessionwiki/docs_pages.py +725 -0
- sessionwiki/doctor.py +113 -0
- sessionwiki/exporters.py +680 -0
- sessionwiki/freshness.py +153 -0
- sessionwiki/graph.py +1058 -0
- sessionwiki/graphify_bridge.py +509 -0
- sessionwiki/ingest_queue.py +101 -0
- sessionwiki/lifecycle.py +167 -0
- sessionwiki/link_checker.py +148 -0
- sessionwiki/lint/__init__.py +170 -0
- sessionwiki/lint/rules/__init__.py +69 -0
- sessionwiki/lint/rules/_helpers.py +119 -0
- sessionwiki/lint/rules/claim_verification.py +44 -0
- sessionwiki/lint/rules/content_freshness.py +41 -0
- sessionwiki/lint/rules/contradiction_detection.py +45 -0
- sessionwiki/lint/rules/duplicate_detection.py +164 -0
- sessionwiki/lint/rules/entity_consistency.py +29 -0
- sessionwiki/lint/rules/frontmatter_completeness.py +46 -0
- sessionwiki/lint/rules/frontmatter_count_consistency.py +61 -0
- sessionwiki/lint/rules/frontmatter_validity.py +71 -0
- sessionwiki/lint/rules/index_sync.py +68 -0
- sessionwiki/lint/rules/link_integrity.py +34 -0
- sessionwiki/lint/rules/orphan_detection.py +43 -0
- sessionwiki/lint/rules/stale_candidates.py +59 -0
- sessionwiki/lint/rules/stale_reference_detection.py +42 -0
- sessionwiki/lint/rules/summary_accuracy.py +36 -0
- sessionwiki/lint/rules/tags_topics_convention.py +58 -0
- sessionwiki/lint/rules/tools_consistency.py +65 -0
- sessionwiki/lock.py +126 -0
- sessionwiki/log_reader.py +129 -0
- sessionwiki/manifest.py +132 -0
- sessionwiki/mcp/__init__.py +29 -0
- sessionwiki/mcp/__main__.py +5 -0
- sessionwiki/mcp/server.py +1061 -0
- sessionwiki/models_page.py +291 -0
- sessionwiki/obsidian_output.py +208 -0
- sessionwiki/opencode/__init__.py +1 -0
- sessionwiki/opencode/autosync.py +102 -0
- sessionwiki/opencode/export.py +419 -0
- sessionwiki/opencode/install.py +152 -0
- sessionwiki/pipeline.py +150 -0
- sessionwiki/project_topics.py +213 -0
- sessionwiki/py.typed +0 -0
- sessionwiki/quarantine.py +273 -0
- sessionwiki/queue.py +29 -0
- sessionwiki/references.py +224 -0
- sessionwiki/render/__init__.py +20 -0
- sessionwiki/render/css.py +988 -0
- sessionwiki/render/docs_css.py +463 -0
- sessionwiki/render/js.py +1337 -0
- sessionwiki/schema.py +270 -0
- sessionwiki/search_facets.py +150 -0
- sessionwiki/search_tree.py +184 -0
- sessionwiki/serve.py +91 -0
- sessionwiki/session_export.py +333 -0
- sessionwiki/skill_installer.py +112 -0
- sessionwiki/sync/__init__.py +13 -0
- sessionwiki/sync/status.py +120 -0
- sessionwiki/synth/__init__.py +14 -0
- sessionwiki/synth/agent_delegate.py +327 -0
- sessionwiki/synth/base.py +125 -0
- sessionwiki/synth/cli_helpers.py +98 -0
- sessionwiki/synth/estimate.py +146 -0
- sessionwiki/synth/ollama.py +379 -0
- sessionwiki/synth/openai_compat.py +325 -0
- sessionwiki/synth/pipeline.py +1489 -0
- sessionwiki/tag_utils.py +62 -0
- sessionwiki/tags.py +359 -0
- sessionwiki/vault.py +347 -0
- sessionwiki/viz_heatmap.py +305 -0
- sessionwiki/viz_tokens.py +429 -0
- sessionwiki/viz_tools.py +241 -0
- sessionwiki/watch.py +154 -0
- sessionwiki/zcode/__init__.py +43 -0
- sessionwiki/zcode/autosync.py +150 -0
- sessionwiki/zcode/export.py +568 -0
- sessionwiki/zcode/install.py +386 -0
- sessionwiki-2.0.0.dist-info/METADATA +303 -0
- sessionwiki-2.0.0.dist-info/RECORD +118 -0
- sessionwiki-2.0.0.dist-info/WHEEL +5 -0
- sessionwiki-2.0.0.dist-info/entry_points.txt +2 -0
- sessionwiki-2.0.0.dist-info/licenses/LICENSE +22 -0
- sessionwiki-2.0.0.dist-info/licenses/NOTICE +14 -0
- sessionwiki-2.0.0.dist-info/top_level.txt +1 -0
sessionwiki/schema.py
ADDED
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
"""Structured entity schema (v0.7 · closes #55).
|
|
2
|
+
|
|
3
|
+
Optional frontmatter schema for entity pages so sessionwiki can render
|
|
4
|
+
queryable info-cards and a sortable `/models/` index. The schema is
|
|
5
|
+
opt-in — pages without `entity_kind: ai-model` are unaffected.
|
|
6
|
+
|
|
7
|
+
Design choices:
|
|
8
|
+
|
|
9
|
+
* **Stdlib-only validation** — we don't pull in pydantic. Validation
|
|
10
|
+
is a plain function returning `(valid_profile, warnings)`. Bad data
|
|
11
|
+
degrades to warnings logged at build time, never crashes the build.
|
|
12
|
+
* **Inline JSON for nested fields** — the existing frontmatter parser
|
|
13
|
+
stores string values; the `model`, `pricing`, and `benchmarks` blocks
|
|
14
|
+
are written as inline JSON (same pattern as `tool_counts` from #63).
|
|
15
|
+
Consumers call `parse_model_profile()` to get a typed dict back.
|
|
16
|
+
* **Forward-compatible benchmark keys** — known keys get nice labels
|
|
17
|
+
in the UI; unknown keys pass through as-is so users can add new
|
|
18
|
+
benchmarks without waiting for a release.
|
|
19
|
+
|
|
20
|
+
Schema (frontmatter):
|
|
21
|
+
|
|
22
|
+
```yaml
|
|
23
|
+
---
|
|
24
|
+
title: "Claude Sonnet 4"
|
|
25
|
+
type: entity
|
|
26
|
+
entity_kind: ai-model
|
|
27
|
+
provider: Anthropic
|
|
28
|
+
model: {"context_window": 200000, "license": "proprietary", "released": "2026-03-18"}
|
|
29
|
+
pricing: {"input_per_1m": 3.00, "output_per_1m": 15.00, "currency": "USD", "effective": "2026-01-15"}
|
|
30
|
+
modalities: [text, vision]
|
|
31
|
+
benchmarks: {"gpqa_diamond": 0.725, "swe_bench": 0.619, "mmlu": 0.887}
|
|
32
|
+
---
|
|
33
|
+
```
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
from __future__ import annotations
|
|
37
|
+
|
|
38
|
+
import json
|
|
39
|
+
from typing import Any, Mapping, Optional, TypedDict
|
|
40
|
+
|
|
41
|
+
ENTITY_KIND_AI_MODEL = "ai-model"
|
|
42
|
+
|
|
43
|
+
# ─── Entity type taxonomy (v1.0, #137) ─────────────────────────────────
|
|
44
|
+
# Seven entity types from the LLM Book design spec (05-metadata-schema.md).
|
|
45
|
+
# Stored in frontmatter as `entity_type: tool` etc.
|
|
46
|
+
|
|
47
|
+
ENTITY_TYPES: tuple[str, ...] = (
|
|
48
|
+
"person", # Individual human
|
|
49
|
+
"org", # Company or organization
|
|
50
|
+
"tool", # Software tool / service
|
|
51
|
+
"concept", # Abstract idea / pattern / framework
|
|
52
|
+
"api", # API or protocol
|
|
53
|
+
"library", # Code library / framework / package
|
|
54
|
+
"project", # Named product / project
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def validate_entity_type(value: str) -> tuple[bool, str]:
|
|
59
|
+
"""Validate an entity_type frontmatter value.
|
|
60
|
+
|
|
61
|
+
Returns (is_valid, message).
|
|
62
|
+
"""
|
|
63
|
+
if not value:
|
|
64
|
+
return False, "entity_type is empty"
|
|
65
|
+
v = value.lower().strip()
|
|
66
|
+
if v in ENTITY_TYPES:
|
|
67
|
+
return True, f"entity_type '{v}' is valid"
|
|
68
|
+
return False, (
|
|
69
|
+
f"entity_type '{value}' is not valid. "
|
|
70
|
+
f"Expected one of: {', '.join(ENTITY_TYPES)}"
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
# Known benchmark keys get pretty labels in the UI. Unknown keys pass
|
|
74
|
+
# through verbatim (forward-compatible).
|
|
75
|
+
KNOWN_BENCHMARKS: dict[str, str] = {
|
|
76
|
+
# Frontier benchmarks (2025-2026)
|
|
77
|
+
"gpqa_diamond": "GPQA Diamond",
|
|
78
|
+
"swe_bench": "SWE-bench",
|
|
79
|
+
"swe_bench_verified": "SWE-bench Verified",
|
|
80
|
+
"aime_2025": "AIME 2025",
|
|
81
|
+
"livecodebench": "LiveCodeBench",
|
|
82
|
+
"arc_agi_2": "ARC-AGI 2",
|
|
83
|
+
# Mid-tier
|
|
84
|
+
"mmlu": "MMLU",
|
|
85
|
+
"mmlu_pro": "MMLU-Pro",
|
|
86
|
+
"humaneval": "HumanEval",
|
|
87
|
+
"hellaswag": "HellaSwag",
|
|
88
|
+
"drop": "DROP",
|
|
89
|
+
"bbh": "BIG-Bench Hard",
|
|
90
|
+
"math_500": "MATH-500",
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
class PricingProfile(TypedDict, total=False):
|
|
95
|
+
input_per_1m: float
|
|
96
|
+
output_per_1m: float
|
|
97
|
+
cache_read_per_1m: float
|
|
98
|
+
cache_write_per_1m: float
|
|
99
|
+
currency: str
|
|
100
|
+
effective: str # ISO date
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
class ModelDetails(TypedDict, total=False):
|
|
104
|
+
context_window: int
|
|
105
|
+
max_output: int
|
|
106
|
+
license: str
|
|
107
|
+
released: str # ISO date
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
class ModelProfile(TypedDict, total=False):
|
|
111
|
+
"""The full structured profile for an `ai-model` entity page."""
|
|
112
|
+
title: str
|
|
113
|
+
provider: str
|
|
114
|
+
model: ModelDetails
|
|
115
|
+
pricing: PricingProfile
|
|
116
|
+
modalities: list[str]
|
|
117
|
+
benchmarks: dict[str, float]
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
# ─── validation ─────────────────────────────────────────────────────────
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _try_parse_json(value: Any) -> Any:
|
|
124
|
+
"""If `value` is a JSON-string dict/list, parse it. Otherwise return as-is."""
|
|
125
|
+
if isinstance(value, str):
|
|
126
|
+
stripped = value.strip()
|
|
127
|
+
if stripped.startswith(("{", "[")):
|
|
128
|
+
try:
|
|
129
|
+
return json.loads(stripped)
|
|
130
|
+
except ValueError:
|
|
131
|
+
return value
|
|
132
|
+
return value
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _coerce_float(value: Any) -> Optional[float]:
|
|
136
|
+
try:
|
|
137
|
+
return float(value)
|
|
138
|
+
except (TypeError, ValueError):
|
|
139
|
+
return None
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _coerce_int(value: Any) -> Optional[int]:
|
|
143
|
+
try:
|
|
144
|
+
return int(value)
|
|
145
|
+
except (TypeError, ValueError):
|
|
146
|
+
return None
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def is_model_entity(meta: Mapping[str, Any]) -> bool:
|
|
150
|
+
"""True if this page is an AI-model entity page (has
|
|
151
|
+
`type: entity` + `entity_kind: ai-model`)."""
|
|
152
|
+
return (
|
|
153
|
+
str(meta.get("type", "")).strip() == "entity"
|
|
154
|
+
and str(meta.get("entity_kind", "")).strip() == ENTITY_KIND_AI_MODEL
|
|
155
|
+
)
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def parse_model_profile(
|
|
159
|
+
meta: Mapping[str, Any],
|
|
160
|
+
) -> tuple[ModelProfile, list[str]]:
|
|
161
|
+
"""Parse + validate a page's model frontmatter.
|
|
162
|
+
|
|
163
|
+
Returns `(profile, warnings)`. `profile` always contains whatever
|
|
164
|
+
fields were parseable, even if the input was partial. `warnings`
|
|
165
|
+
contains human-readable messages for any field that failed
|
|
166
|
+
validation — the caller logs them and keeps building.
|
|
167
|
+
"""
|
|
168
|
+
warnings: list[str] = []
|
|
169
|
+
profile: ModelProfile = {}
|
|
170
|
+
|
|
171
|
+
title = meta.get("title")
|
|
172
|
+
if title:
|
|
173
|
+
profile["title"] = str(title)
|
|
174
|
+
|
|
175
|
+
provider = meta.get("provider")
|
|
176
|
+
if provider:
|
|
177
|
+
profile["provider"] = str(provider)
|
|
178
|
+
|
|
179
|
+
# Nested blocks are inline JSON in the frontmatter
|
|
180
|
+
model_block = _try_parse_json(meta.get("model"))
|
|
181
|
+
if isinstance(model_block, dict):
|
|
182
|
+
details: ModelDetails = {}
|
|
183
|
+
cw = _coerce_int(model_block.get("context_window"))
|
|
184
|
+
if cw is not None and cw > 0:
|
|
185
|
+
details["context_window"] = cw
|
|
186
|
+
elif model_block.get("context_window") is not None:
|
|
187
|
+
warnings.append("model.context_window must be a positive integer")
|
|
188
|
+
|
|
189
|
+
mo = _coerce_int(model_block.get("max_output"))
|
|
190
|
+
if mo is not None and mo > 0:
|
|
191
|
+
details["max_output"] = mo
|
|
192
|
+
elif model_block.get("max_output") is not None:
|
|
193
|
+
warnings.append("model.max_output must be a positive integer")
|
|
194
|
+
|
|
195
|
+
for k in ("license", "released"):
|
|
196
|
+
v = model_block.get(k)
|
|
197
|
+
if v:
|
|
198
|
+
details[k] = str(v) # type: ignore[literal-required]
|
|
199
|
+
|
|
200
|
+
if details:
|
|
201
|
+
profile["model"] = details
|
|
202
|
+
elif model_block:
|
|
203
|
+
warnings.append("model block must be a JSON object")
|
|
204
|
+
|
|
205
|
+
pricing_block = _try_parse_json(meta.get("pricing"))
|
|
206
|
+
if isinstance(pricing_block, dict):
|
|
207
|
+
pricing: PricingProfile = {}
|
|
208
|
+
for k in ("input_per_1m", "output_per_1m",
|
|
209
|
+
"cache_read_per_1m", "cache_write_per_1m"):
|
|
210
|
+
v = pricing_block.get(k)
|
|
211
|
+
if v is None:
|
|
212
|
+
continue
|
|
213
|
+
f = _coerce_float(v)
|
|
214
|
+
if f is None or f < 0:
|
|
215
|
+
warnings.append(f"pricing.{k} must be a non-negative number")
|
|
216
|
+
continue
|
|
217
|
+
pricing[k] = f # type: ignore[literal-required]
|
|
218
|
+
if pricing_block.get("currency"):
|
|
219
|
+
pricing["currency"] = str(pricing_block["currency"])
|
|
220
|
+
if pricing_block.get("effective"):
|
|
221
|
+
pricing["effective"] = str(pricing_block["effective"])
|
|
222
|
+
if pricing:
|
|
223
|
+
profile["pricing"] = pricing
|
|
224
|
+
elif pricing_block:
|
|
225
|
+
warnings.append("pricing block must be a JSON object")
|
|
226
|
+
|
|
227
|
+
modalities = meta.get("modalities")
|
|
228
|
+
if isinstance(modalities, list):
|
|
229
|
+
profile["modalities"] = [str(m) for m in modalities if m]
|
|
230
|
+
elif isinstance(modalities, str) and modalities:
|
|
231
|
+
# Comma-separated fallback
|
|
232
|
+
profile["modalities"] = [
|
|
233
|
+
m.strip() for m in modalities.strip("[]").split(",") if m.strip()
|
|
234
|
+
]
|
|
235
|
+
|
|
236
|
+
benchmarks_block = _try_parse_json(meta.get("benchmarks"))
|
|
237
|
+
if isinstance(benchmarks_block, dict):
|
|
238
|
+
benches: dict[str, float] = {}
|
|
239
|
+
for k, v in benchmarks_block.items():
|
|
240
|
+
f = _coerce_float(v)
|
|
241
|
+
if f is None:
|
|
242
|
+
warnings.append(f"benchmarks.{k} must be a number")
|
|
243
|
+
continue
|
|
244
|
+
if f < 0 or f > 1:
|
|
245
|
+
warnings.append(
|
|
246
|
+
f"benchmarks.{k}={f} should be in [0, 1] (fraction, not percent)"
|
|
247
|
+
)
|
|
248
|
+
continue
|
|
249
|
+
benches[str(k)] = f
|
|
250
|
+
if benches:
|
|
251
|
+
profile["benchmarks"] = benches
|
|
252
|
+
elif benchmarks_block:
|
|
253
|
+
warnings.append("benchmarks block must be a JSON object")
|
|
254
|
+
|
|
255
|
+
return profile, warnings
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def benchmark_label(key: str) -> str:
|
|
259
|
+
"""Return a human-readable label for a known benchmark key, or the
|
|
260
|
+
raw key (with underscores → spaces + titlecased) for unknown ones."""
|
|
261
|
+
if key in KNOWN_BENCHMARKS:
|
|
262
|
+
return KNOWN_BENCHMARKS[key]
|
|
263
|
+
return key.replace("_", " ").title()
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def format_price(value: float, currency: str = "USD") -> str:
|
|
267
|
+
"""Format a per-1M-token price. Always 2 decimals, currency prefix."""
|
|
268
|
+
symbols = {"USD": "$", "EUR": "€", "GBP": "£"}
|
|
269
|
+
prefix = symbols.get(currency.upper(), f"{currency} ")
|
|
270
|
+
return f"{prefix}{value:.2f}"
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
"""Search facet enrichment (v1.0 · #161).
|
|
2
|
+
|
|
3
|
+
Adds confidence-weighted ranking and facet filters (entity_type,
|
|
4
|
+
lifecycle) to the static-site search index.
|
|
5
|
+
|
|
6
|
+
Builds on the chunked search index from #47. Each entry gains:
|
|
7
|
+
- ``confidence``: float 0.0-1.0 (from page frontmatter)
|
|
8
|
+
- ``lifecycle``: state string (draft/reviewed/verified/stale/archived)
|
|
9
|
+
- ``entity_type``: person/org/tool/concept/api/library/project
|
|
10
|
+
- ``tags``: list[str] (filtered to non-noise tags)
|
|
11
|
+
|
|
12
|
+
The facet summary (counts per value) is written alongside the index
|
|
13
|
+
so the client can render filter checkboxes without scanning the full
|
|
14
|
+
index first.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
from typing import Any
|
|
20
|
+
|
|
21
|
+
# Shared tag parser + NOISE_TAGS live in sessionwiki.tag_utils so
|
|
22
|
+
# sessionwiki/categories.py uses the same implementation.
|
|
23
|
+
from sessionwiki.tag_utils import NOISE_TAGS, parse_tags_field as _parse_tags_field
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _parse_confidence(raw: Any) -> float:
|
|
27
|
+
"""Parse a confidence field, return 0.0 if missing/invalid."""
|
|
28
|
+
if raw is None or raw == "":
|
|
29
|
+
return 0.0
|
|
30
|
+
try:
|
|
31
|
+
c = float(raw)
|
|
32
|
+
except (ValueError, TypeError):
|
|
33
|
+
return 0.0
|
|
34
|
+
return max(0.0, min(1.0, c))
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def enrich_entry(entry: dict[str, Any], meta: dict[str, Any]) -> dict[str, Any]:
|
|
38
|
+
"""Add facet fields to a search-index entry.
|
|
39
|
+
|
|
40
|
+
Modifies ``entry`` in place and returns it.
|
|
41
|
+
"""
|
|
42
|
+
entry["confidence"] = _parse_confidence(meta.get("confidence"))
|
|
43
|
+
entry["lifecycle"] = str(meta.get("lifecycle", "")).lower()
|
|
44
|
+
entry["entity_type"] = str(meta.get("entity_type", "")).lower()
|
|
45
|
+
entry["tags"] = _parse_tags_field(meta.get("tags", ""))
|
|
46
|
+
return entry
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def aggregate_facets(entries: list[dict[str, Any]]) -> dict[str, dict[str, int]]:
|
|
50
|
+
"""Count entries by each facet value. Returns::
|
|
51
|
+
|
|
52
|
+
{
|
|
53
|
+
"entity_type": {"tool": 12, "concept": 8, ...},
|
|
54
|
+
"lifecycle": {"draft": 5, "verified": 20, ...},
|
|
55
|
+
"tags": {"flutter": 3, "python": 7, ...},
|
|
56
|
+
"confidence": {"high": N, "medium": N, "low": N, "none": N},
|
|
57
|
+
}
|
|
58
|
+
"""
|
|
59
|
+
counts = {
|
|
60
|
+
"entity_type": {},
|
|
61
|
+
"lifecycle": {},
|
|
62
|
+
"tags": {},
|
|
63
|
+
"confidence": {},
|
|
64
|
+
}
|
|
65
|
+
for entry in entries:
|
|
66
|
+
et = entry.get("entity_type", "")
|
|
67
|
+
if et:
|
|
68
|
+
counts["entity_type"][et] = counts["entity_type"].get(et, 0) + 1
|
|
69
|
+
|
|
70
|
+
lc = entry.get("lifecycle", "")
|
|
71
|
+
if lc:
|
|
72
|
+
counts["lifecycle"][lc] = counts["lifecycle"].get(lc, 0) + 1
|
|
73
|
+
|
|
74
|
+
for tag in entry.get("tags", []):
|
|
75
|
+
counts["tags"][tag] = counts["tags"].get(tag, 0) + 1
|
|
76
|
+
|
|
77
|
+
conf = entry.get("confidence", 0.0)
|
|
78
|
+
if conf >= 0.8:
|
|
79
|
+
bucket = "high"
|
|
80
|
+
elif conf >= 0.5:
|
|
81
|
+
bucket = "medium"
|
|
82
|
+
elif conf > 0:
|
|
83
|
+
bucket = "low"
|
|
84
|
+
else:
|
|
85
|
+
bucket = "none"
|
|
86
|
+
counts["confidence"][bucket] = counts["confidence"].get(bucket, 0) + 1
|
|
87
|
+
|
|
88
|
+
return counts
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def rank_by_confidence(
|
|
92
|
+
entries: list[dict[str, Any]],
|
|
93
|
+
query: str = "",
|
|
94
|
+
*,
|
|
95
|
+
confidence_weight: float = 0.3,
|
|
96
|
+
) -> list[dict[str, Any]]:
|
|
97
|
+
"""Return entries sorted by a combined relevance + confidence score.
|
|
98
|
+
|
|
99
|
+
If ``query`` is empty, sorts by confidence alone. If provided, does
|
|
100
|
+
a simple substring match on title and body, weighted by confidence.
|
|
101
|
+
"""
|
|
102
|
+
q = query.lower().strip()
|
|
103
|
+
|
|
104
|
+
def _score(entry: dict[str, Any]) -> float:
|
|
105
|
+
conf = entry.get("confidence", 0.0)
|
|
106
|
+
if not q:
|
|
107
|
+
return conf
|
|
108
|
+
|
|
109
|
+
title = str(entry.get("title", "")).lower()
|
|
110
|
+
body = str(entry.get("body", "")).lower()
|
|
111
|
+
tags = entry.get("tags", [])
|
|
112
|
+
|
|
113
|
+
relevance = 0.0
|
|
114
|
+
if q in title:
|
|
115
|
+
relevance += 1.0
|
|
116
|
+
if q in body:
|
|
117
|
+
relevance += 0.5
|
|
118
|
+
if any(q in t for t in tags):
|
|
119
|
+
relevance += 0.3
|
|
120
|
+
|
|
121
|
+
return relevance * (1.0 - confidence_weight) + conf * confidence_weight
|
|
122
|
+
|
|
123
|
+
return sorted(entries, key=_score, reverse=True)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def filter_entries(
|
|
127
|
+
entries: list[dict[str, Any]],
|
|
128
|
+
*,
|
|
129
|
+
entity_types: list[str] | None = None,
|
|
130
|
+
lifecycles: list[str] | None = None,
|
|
131
|
+
tags: list[str] | None = None,
|
|
132
|
+
min_confidence: float = 0.0,
|
|
133
|
+
max_confidence: float = 1.0,
|
|
134
|
+
) -> list[dict[str, Any]]:
|
|
135
|
+
"""Filter entries by facet values. Empty filter lists = no filter."""
|
|
136
|
+
result = []
|
|
137
|
+
for entry in entries:
|
|
138
|
+
if entity_types and entry.get("entity_type", "") not in entity_types:
|
|
139
|
+
continue
|
|
140
|
+
if lifecycles and entry.get("lifecycle", "") not in lifecycles:
|
|
141
|
+
continue
|
|
142
|
+
if tags:
|
|
143
|
+
entry_tags = set(entry.get("tags", []))
|
|
144
|
+
if not entry_tags & set(tags):
|
|
145
|
+
continue
|
|
146
|
+
conf = entry.get("confidence", 0.0)
|
|
147
|
+
if not (min_confidence <= conf <= max_confidence):
|
|
148
|
+
continue
|
|
149
|
+
result.append(entry)
|
|
150
|
+
return result
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
"""Tree-aware search routing (v1.2.0 · #53).
|
|
2
|
+
|
|
3
|
+
TreeSearch reports +25% MRR on academic papers and 37× faster than
|
|
4
|
+
embedding search when ≥ 30 % of the corpus has heading depth ≥ 3. Below
|
|
5
|
+
that threshold, **flat mode is faster and gives better results**, so
|
|
6
|
+
this module's job is to decide which mode to use and expose the reason
|
|
7
|
+
to the client.
|
|
8
|
+
|
|
9
|
+
Flow
|
|
10
|
+
----
|
|
11
|
+
1. `sessionwiki build` runs `annotate_entry_headings(entry, body)` on every
|
|
12
|
+
session / page / entity entry it puts into `search-index.json`. That
|
|
13
|
+
injects `heading_max_depth` + `heading_count_by_depth` into the
|
|
14
|
+
entry so the client never has to re-parse markdown.
|
|
15
|
+
2. `sessionwiki build` calls `decide_search_mode(entries, override=...)`
|
|
16
|
+
after collecting all entries. Result lands at the top level of
|
|
17
|
+
`search-index.json` under `_mode` + `_tree_eligible_ratio`.
|
|
18
|
+
3. The client reads `_mode` on first load and branches:
|
|
19
|
+
- ``flat`` → existing behaviour, exact-match first then body match.
|
|
20
|
+
- ``tree`` → match by title first, then best-first walk into the
|
|
21
|
+
highest-depth heading that matched.
|
|
22
|
+
|
|
23
|
+
Manual override: `sessionwiki build --search-mode {auto,tree,flat}`. Default
|
|
24
|
+
is ``auto`` — the heuristic picks.
|
|
25
|
+
|
|
26
|
+
Public API
|
|
27
|
+
----------
|
|
28
|
+
- :data:`TREE_ELIGIBLE_DEPTH` — minimum heading depth that counts as "deep"
|
|
29
|
+
- :data:`TREE_MODE_THRESHOLD` — fraction of deep pages that flips to tree
|
|
30
|
+
- :data:`SEARCH_MODES` — the valid mode tuple
|
|
31
|
+
- :func:`heading_depths` — pure regex scan for per-body depth stats
|
|
32
|
+
- :func:`annotate_entry_headings` — mutate an entry dict in place
|
|
33
|
+
- :func:`decide_search_mode` — pick mode from corpus + optional override
|
|
34
|
+
- :func:`search_index_footer_badge` — short label the palette footer shows
|
|
35
|
+
|
|
36
|
+
Design notes
|
|
37
|
+
------------
|
|
38
|
+
- **Stdlib only.** No markdown parser — a regex over ``^(#+) `` is fast
|
|
39
|
+
and accurate enough; pages aren't exotic markdown.
|
|
40
|
+
- **Client-side cost is one extra key per entry.** We don't ship
|
|
41
|
+
heading text into the index (keeps chunk sizes flat) — only counts.
|
|
42
|
+
The client reads the target page's HTML directly when the user
|
|
43
|
+
expands a tree hit.
|
|
44
|
+
- **Override beats heuristic.** ``--search-mode tree`` forces tree even
|
|
45
|
+
on a shallow corpus, useful when testing a themed subset.
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
from __future__ import annotations
|
|
49
|
+
|
|
50
|
+
import re
|
|
51
|
+
from typing import Any, Iterable, Literal, Mapping, Optional, TypedDict
|
|
52
|
+
|
|
53
|
+
# ─── Constants ─────────────────────────────────────────────────────────
|
|
54
|
+
|
|
55
|
+
# A heading counts as "deep" at ## or below (h2, h3, h4, ...). The
|
|
56
|
+
# benchmark cited in issue #53 is h3+; we use ≥ 3 to match.
|
|
57
|
+
TREE_ELIGIBLE_DEPTH = 3
|
|
58
|
+
|
|
59
|
+
# Fraction of corpus pages with depth ≥ TREE_ELIGIBLE_DEPTH above which
|
|
60
|
+
# tree mode wins. Below that, flat is both faster and produces better
|
|
61
|
+
# ranking. 30% is the TreeSearch-paper threshold.
|
|
62
|
+
TREE_MODE_THRESHOLD = 0.30
|
|
63
|
+
|
|
64
|
+
SearchMode = Literal["flat", "tree", "auto"]
|
|
65
|
+
SEARCH_MODES: tuple[SearchMode, ...] = ("flat", "tree", "auto")
|
|
66
|
+
DEFAULT_SEARCH_MODE: SearchMode = "auto"
|
|
67
|
+
|
|
68
|
+
# Max heading depth we bother recording — h6 is already "use a paragraph
|
|
69
|
+
# instead". Deeper headings still count but bucket under depth 6.
|
|
70
|
+
_MAX_DEPTH_BUCKET = 6
|
|
71
|
+
|
|
72
|
+
# ``^#+ `` at line start. Match up to 10 `#` so we can detect
|
|
73
|
+
# "not actually a heading, just a really long fence" cases. We use
|
|
74
|
+
# `[ \t]+` (not `\s+`) so a bare `#\n` can't match across the newline
|
|
75
|
+
# and get treated as depth-1.
|
|
76
|
+
_HEADING_RE = re.compile(r"^(#{1,10})[ \t]+\S", re.MULTILINE)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
# ─── Per-body heading stats ────────────────────────────────────────────
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class HeadingStats(TypedDict):
|
|
83
|
+
"""Shape injected into each search-index entry."""
|
|
84
|
+
|
|
85
|
+
heading_max_depth: int # 0 if no headings at all
|
|
86
|
+
heading_count_by_depth: dict[int, int]
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def heading_depths(body: str) -> HeadingStats:
|
|
90
|
+
"""Scan a markdown body and return ``(max_depth, count_by_depth)``.
|
|
91
|
+
|
|
92
|
+
A depth of 1 is ``# H1`` (the page title, often stripped before
|
|
93
|
+
indexing). 2 is ``##``, 3 is ``###``, etc. Deeper than
|
|
94
|
+
:data:`_MAX_DEPTH_BUCKET` is bucketed into the last depth so we
|
|
95
|
+
don't carry tail noise.
|
|
96
|
+
"""
|
|
97
|
+
counts: dict[int, int] = {}
|
|
98
|
+
max_depth = 0
|
|
99
|
+
if not body:
|
|
100
|
+
return {"heading_max_depth": 0, "heading_count_by_depth": counts}
|
|
101
|
+
|
|
102
|
+
for match in _HEADING_RE.finditer(body):
|
|
103
|
+
raw_hashes = len(match.group(1))
|
|
104
|
+
depth = min(raw_hashes, _MAX_DEPTH_BUCKET)
|
|
105
|
+
counts[depth] = counts.get(depth, 0) + 1
|
|
106
|
+
if depth > max_depth:
|
|
107
|
+
max_depth = depth
|
|
108
|
+
|
|
109
|
+
return {
|
|
110
|
+
"heading_max_depth": max_depth,
|
|
111
|
+
"heading_count_by_depth": counts,
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def annotate_entry_headings(entry: dict[str, Any], body: str) -> None:
|
|
116
|
+
"""Mutate ``entry`` in place to add ``heading_max_depth`` and
|
|
117
|
+
``heading_count_by_depth`` — the two keys the client + the mode
|
|
118
|
+
decider consume.
|
|
119
|
+
"""
|
|
120
|
+
stats = heading_depths(body)
|
|
121
|
+
entry["heading_max_depth"] = stats["heading_max_depth"]
|
|
122
|
+
# JSON serialization: dict keys must be strings. Store as str keys
|
|
123
|
+
# so ``json.dumps(entry)`` doesn't need a custom encoder.
|
|
124
|
+
entry["heading_count_by_depth"] = {
|
|
125
|
+
str(k): v for k, v in stats["heading_count_by_depth"].items()
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
# ─── Mode decision ─────────────────────────────────────────────────────
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _deep_ratio(entries: Iterable[Mapping[str, Any]]) -> tuple[int, int, float]:
|
|
133
|
+
"""Return ``(deep_pages, total_pages, ratio)``."""
|
|
134
|
+
total = 0
|
|
135
|
+
deep = 0
|
|
136
|
+
for entry in entries:
|
|
137
|
+
depth = int(entry.get("heading_max_depth", 0))
|
|
138
|
+
total += 1
|
|
139
|
+
if depth >= TREE_ELIGIBLE_DEPTH:
|
|
140
|
+
deep += 1
|
|
141
|
+
ratio = deep / total if total else 0.0
|
|
142
|
+
return deep, total, ratio
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def decide_search_mode(
|
|
146
|
+
entries: Iterable[Mapping[str, Any]],
|
|
147
|
+
*,
|
|
148
|
+
override: Optional[str] = None,
|
|
149
|
+
) -> tuple[SearchMode, float]:
|
|
150
|
+
"""Pick the search mode (``"tree"`` or ``"flat"``) for a corpus.
|
|
151
|
+
|
|
152
|
+
Returns ``(mode, deep_ratio)`` so the caller can surface the ratio
|
|
153
|
+
to the user via the palette footer.
|
|
154
|
+
|
|
155
|
+
``override`` accepts the same values as the CLI flag: ``"auto"``,
|
|
156
|
+
``"tree"``, or ``"flat"``. Unknown strings fall back to auto +
|
|
157
|
+
heuristic; ``None`` means "no flag passed, use auto".
|
|
158
|
+
"""
|
|
159
|
+
normalized = (override or DEFAULT_SEARCH_MODE).strip().lower()
|
|
160
|
+
deep, total, ratio = _deep_ratio(entries)
|
|
161
|
+
|
|
162
|
+
if normalized == "tree":
|
|
163
|
+
return "tree", ratio
|
|
164
|
+
if normalized == "flat":
|
|
165
|
+
return "flat", ratio
|
|
166
|
+
if normalized not in SEARCH_MODES:
|
|
167
|
+
# Typo — warn via fallback. The caller prints; we just decide.
|
|
168
|
+
normalized = DEFAULT_SEARCH_MODE
|
|
169
|
+
|
|
170
|
+
# Auto path: flip to tree only if the corpus clears the threshold.
|
|
171
|
+
if total == 0:
|
|
172
|
+
return "flat", ratio
|
|
173
|
+
return ("tree" if ratio >= TREE_MODE_THRESHOLD else "flat", ratio)
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def search_index_footer_badge(
|
|
177
|
+
mode: SearchMode, ratio: float
|
|
178
|
+
) -> str:
|
|
179
|
+
"""Short text the search palette displays so users see which mode
|
|
180
|
+
the corpus landed on today."""
|
|
181
|
+
pct = int(round(ratio * 100))
|
|
182
|
+
if mode == "tree":
|
|
183
|
+
return f"tree mode · {pct}% deep pages"
|
|
184
|
+
return f"flat mode · {pct}% deep pages"
|
sessionwiki/serve.py
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
"""Local HTTP server for the built sessionwiki site.
|
|
2
|
+
|
|
3
|
+
Uses only Python stdlib. Binds to 127.0.0.1 by default so nothing is exposed
|
|
4
|
+
to the network unless the user explicitly passes --host 0.0.0.0.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import http.server
|
|
10
|
+
import socketserver
|
|
11
|
+
import webbrowser
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class _QuietHandler(http.server.SimpleHTTPRequestHandler):
|
|
16
|
+
"""Like SimpleHTTPRequestHandler but with prettier logs and a branded
|
|
17
|
+
404 response that pulls ``site/404.html`` (closes #387 U8) instead of
|
|
18
|
+
falling back to the stdlib's plain-text error page."""
|
|
19
|
+
|
|
20
|
+
def log_message(self, format: str, *args) -> None: # noqa: A002
|
|
21
|
+
# Suppress per-request logs for a cleaner terminal.
|
|
22
|
+
return
|
|
23
|
+
|
|
24
|
+
def send_error(self, code: int, message: str | None = None,
|
|
25
|
+
explain: str | None = None) -> None:
|
|
26
|
+
"""Override the default error page so 404s pick up the branded
|
|
27
|
+
``404.html`` shipped by ``sessionwiki build``. We deliberately keep the
|
|
28
|
+
404 status code intact — the page is the *body* of the 404 response,
|
|
29
|
+
not a redirect — so crawlers still see the right HTTP code.
|
|
30
|
+
|
|
31
|
+
Falls back to the stdlib default for anything other than 404, or
|
|
32
|
+
when ``404.html`` is missing (e.g. a partially-built site)."""
|
|
33
|
+
if code == 404:
|
|
34
|
+
try:
|
|
35
|
+
# #py-m2 (#588): no longer relies on os.chdir(). The
|
|
36
|
+
# SimpleHTTPRequestHandler's `directory` arg holds the
|
|
37
|
+
# site root; we read 404.html from there explicitly.
|
|
38
|
+
site_root = getattr(self, "directory", None)
|
|
39
|
+
err_page = (Path(site_root) / "404.html") if site_root else Path("404.html")
|
|
40
|
+
with open(err_page, "rb") as f:
|
|
41
|
+
body = f.read()
|
|
42
|
+
self.send_response(404, message)
|
|
43
|
+
self.send_header("Content-Type", "text/html; charset=utf-8")
|
|
44
|
+
self.send_header("Content-Length", str(len(body)))
|
|
45
|
+
self.end_headers()
|
|
46
|
+
self.wfile.write(body)
|
|
47
|
+
return
|
|
48
|
+
except (FileNotFoundError, OSError):
|
|
49
|
+
# 404.html doesn't exist — fall through to default behavior.
|
|
50
|
+
pass
|
|
51
|
+
super().send_error(code, message, explain)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class _ReusableTCPServer(socketserver.TCPServer):
|
|
55
|
+
allow_reuse_address = True
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def serve_site(
|
|
59
|
+
directory: Path,
|
|
60
|
+
port: int = 8765,
|
|
61
|
+
host: str = "127.0.0.1",
|
|
62
|
+
open_browser: bool = False,
|
|
63
|
+
) -> int:
|
|
64
|
+
directory = directory.expanduser().resolve()
|
|
65
|
+
if not directory.exists():
|
|
66
|
+
print(f"error: {directory} does not exist. Run `sessionwiki build` first.")
|
|
67
|
+
return 2
|
|
68
|
+
# #py-m2 (#588): use SimpleHTTPRequestHandler's `directory=` kwarg
|
|
69
|
+
# (Python 3.7+) instead of mutating global cwd. The previous
|
|
70
|
+
# `os.chdir(directory)` call leaked process state — every test
|
|
71
|
+
# using this function had to remember to chdir back, and
|
|
72
|
+
# concurrent serve_site calls in tests would race.
|
|
73
|
+
handler_factory = lambda *a, **kw: _QuietHandler(*a, directory=str(directory), **kw)
|
|
74
|
+
url = f"http://{host}:{port}/"
|
|
75
|
+
print(f"==> Serving {directory} at {url}")
|
|
76
|
+
print(" Press Ctrl+C to stop.")
|
|
77
|
+
try:
|
|
78
|
+
with _ReusableTCPServer((host, port), handler_factory) as httpd:
|
|
79
|
+
if open_browser:
|
|
80
|
+
try:
|
|
81
|
+
webbrowser.open(url)
|
|
82
|
+
except Exception:
|
|
83
|
+
pass
|
|
84
|
+
try:
|
|
85
|
+
httpd.serve_forever()
|
|
86
|
+
except KeyboardInterrupt:
|
|
87
|
+
print("\n stopped.")
|
|
88
|
+
except OSError as e:
|
|
89
|
+
print(f"error: could not bind {host}:{port}: {e}")
|
|
90
|
+
return 1
|
|
91
|
+
return 0
|