handrolled 0.1.0__py3-none-any.whl

handrolled/__init__.py

@@ -0,0 +1,26 @@
+"""Hand-rolled, stdlib-only replacements for things you would normally pip install.
+
+Every module here exists for the same reason: the obvious solution adds a
+dependency, and the dependency is bigger than the problem.
+
+* :mod:`handrolled.text` — CJK bigrams, FTS5 match expressions and a
+  YAML-subset front matter reader (instead of jieba, a query builder and PyYAML)
+* :mod:`handrolled.webdav` — a WebDAV client speaking the RFC 4918 subset that
+  actually gets used (instead of requests + webdavclient3)
+* :mod:`handrolled.mounts` — network mount inspection that reports what is
+  really mounted, not its autofs trigger (instead of parsing ``mount`` output)
+* :mod:`handrolled.pages` + :mod:`handrolled.importers` — turn another tool's
+  export (mem0, Obsidian, Notion, basic-memory, JSON/JSONL) into Markdown pages
+* :mod:`handrolled.rerank`, :mod:`handrolled.deeprank`,
+  :mod:`handrolled.embed_onnx` — optional ranking helpers that degrade to
+  ``None`` when their model or dependency is absent
+
+Nothing here imports a third-party package at module import time; the optional
+ones import inside functions and return ``None`` when the dependency is missing.
+"""
+
+from __future__ import annotations
+
+__version__ = "0.1.0"
+
+__all__ = ["__version__"]

handrolled/deeprank.py

@@ -0,0 +1,183 @@
+"""深池 LLM 重排（`deep=true` 慢车道）。
+
+为什么单独一层而不是改主链路：
+* 主链路要快（p95 < 1s），LLM 一次 5~15s，只能按需触发；
+* 需要「看得到更远的候选」——先把候选扩到 40 页，再让模型挑；
+* 失败必须静默降级：API 挂了就退回普通排序，绝不影响检索可用性。
+
+一次 33 题评测：普通 25/33 → 深池 + 挑 1 页 = 26/33，代价 p50 8.4s。
+单次用量约 3.7k 输入 token / 120 输出 token。
+
+注意一个实测过的失败模式：候选多时模型会报出**超出范围的编号**（例如只有 30 页却答 31）。
+所以解析不到合法编号时，会用「前 10 页」重试一次，仍然失败才降级。
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import urllib.request
+from typing import Sequence
+
+#: This is an opt-in slow lane: nothing below runs unless the deployment says
+#: where to send the request and with which credentials.  No vendor endpoint,
+#: no key file and no log path is assumed.
+DEFAULT_MODEL = ""
+DEFAULT_BASE = ""
+CRED_PATH = os.environ.get("DSH_KB_DEEP_CREDENTIALS") or ""
+
+LAST_RAW = ""
+LAST_ERROR = ""
+LOG_PATH = os.environ.get("DSH_KB_DEEP_LOG") or ""
+
+PROMPT = """用户问题：%s
+
+下面是知识库候选页面（编号 1-%d，含该页最相关片段）。请挑出最可能回答了该问题的 %d 个页面，
+按可能性从高到低输出编号，逗号分隔，只输出编号（必须是 1 到 %d 之间的整数）。
+
+%s"""
+
+
+def _api_key() -> str | None:
+    """API key: ``DSH_KB_DEEP_API_KEY`` first, then an optional credentials file.
+
+    The file is whatever ``DSH_KB_DEEP_CREDENTIALS`` points at and only has to
+    contain ``DEEP_API_KEY`` (top level or under ``refs``).  YAML is used when
+    PyYAML happens to be importable, otherwise one line is matched with a regex —
+    the MCP server interpreter may not have yaml installed.
+    """
+    global LAST_ERROR
+    key = (os.environ.get("DSH_KB_DEEP_API_KEY") or "").strip()
+    if key:
+        return key
+    if not CRED_PATH:
+        LAST_ERROR = "no-key"
+        return None
+    try:
+        raw = open(CRED_PATH, encoding="utf-8").read()
+    except Exception as exc:
+        LAST_ERROR = "cred-read: %s" % type(exc).__name__
+        return None
+    try:
+        import yaml
+
+        data = yaml.safe_load(raw) or {}
+        refs = data.get("refs") or data
+        value = refs.get("DEEP_API_KEY")
+        if value:
+            return str(value)
+    except Exception:
+        pass
+    m = re.search(r"^\s*DEEP_API_KEY\s*:\s*(\S+)", raw, re.M)
+    if m:
+        return m.group(1).strip().strip(chr(34)).strip(chr(39))
+    LAST_ERROR = "no-key"
+    return None
+
+
+def _endpoint() -> str:
+    """OpenAI-compatible base URL, without the trailing slash."""
+    return (os.environ.get("DSH_KB_DEEP_ENDPOINT") or DEFAULT_BASE).rstrip("/")
+
+
+def _parse_order(text: str, n: int, pick: int) -> list[int]:
+    """把「3,1,7」解析成 1 基下标；越界与重复丢掉。"""
+    out: list[int] = []
+    for raw in re.findall(r"\d+", text or ""):
+        try:
+            idx = int(raw)
+        except ValueError:
+            continue
+        if 1 <= idx <= n and idx not in out:
+            out.append(idx)
+        if len(out) >= pick:
+            break
+    return out
+
+
+def _request(query: str, pages: Sequence[tuple[str, str]], *, key: str, model: str,
+             pick: int, timeout: float) -> tuple[list[int], str]:
+    blocks = ["[%d] %s" % (i, path.split("/")[-1]) + chr(10) + (body or "")[:320]
+              for i, (path, body) in enumerate(pages, 1)]
+    prompt = PROMPT % (query, len(pages), pick, len(pages), chr(10) + chr(10).join(blocks))
+    body = json.dumps({"model": model, "temperature": 0, "max_tokens": 400,
+                       "messages": [{"role": "user", "content": prompt}]}).encode("utf-8")
+    req = urllib.request.Request(_endpoint() + "/chat/completions", data=body,
+                                 headers={"Content-Type": "application/json",
+                                          "Authorization": "Bearer " + key})
+    with urllib.request.urlopen(req, timeout=timeout) as resp:
+        data = json.loads(resp.read().decode("utf-8"))
+    text = ((data.get("choices") or [{}])[0].get("message") or {}).get("content") or ""
+    return _parse_order(text, len(pages), pick), text
+
+
+def _log_failure(query: str, error: str, raw: str) -> None:
+    """降级时留一行证据：MCP 进程看不到 stdout，只有文件能查。
+
+    Nothing is written unless ``DSH_KB_DEEP_LOG`` is set — a library has no
+    business picking a log location on someone else's machine.
+    """
+    if not LOG_PATH:
+        return
+    try:
+        from datetime import datetime
+
+        line = "%s\t%s\t%s\t%s" % (
+            datetime.now().isoformat(timespec="seconds"),
+            error,
+            (query or "")[:40].replace("\t", " "),
+            (raw or "").replace("\t", " ")[:40],
+        )
+        with open(LOG_PATH, "a", encoding="utf-8") as fh:
+            fh.write(line + chr(10))
+    except Exception:
+        pass
+
+
+def pick_pages(
+    query: str,
+    pages: Sequence[tuple[str, str]],
+    *,
+    model: str = DEFAULT_MODEL,
+    pick: int = 1,
+    timeout: float = 25.0,
+) -> list[int] | None:
+    """让模型从候选页面里挑 pick 个；失败返回 None。
+
+    ``pages`` 是 [(path, passage)]，passage 由调用方切好（避免这里再读盘）。
+    """
+    global LAST_RAW, LAST_ERROR
+    LAST_RAW, LAST_ERROR = "", ""
+    if not pages:
+        return []
+    key = _api_key()
+    if not key:
+        LAST_ERROR = "no-key"
+        return None
+    if not _endpoint() or not model:
+        LAST_ERROR = "no-endpoint-or-model"
+        return None
+    for attempt, subset in enumerate((list(pages), list(pages)[:10])):
+        if attempt and len(subset) >= len(pages):
+            break
+        try:
+            order, text = _request(query, subset, key=key, model=model, pick=pick,
+                                   timeout=timeout if attempt == 0 else min(timeout, 15.0))
+        except Exception as exc:  # noqa: BLE001 - 任何失败都降级
+            LAST_ERROR = "%s: %s" % (type(exc).__name__, str(exc)[:120])
+            continue
+        LAST_RAW = (text or "").strip()[:80]
+        if order:
+            return order
+        LAST_ERROR = "unparsable"
+    _log_failure(query, LAST_ERROR, LAST_RAW)
+    return None
+
+
+def reorder(items: Sequence, order: Sequence[int]) -> list:
+    """按 1 基下标把挑中的提到最前，其余保序。"""
+    idx = [i - 1 for i in order if 0 < i <= len(items)]
+    picked = [items[i] for i in idx]
+    rest = [x for j, x in enumerate(items) if j not in set(idx)]
+    return picked + rest

handrolled/embed_onnx.py

@@ -0,0 +1,168 @@
+"""ONNX 嵌入后端（Qwen3-Embedding-0.6B int8）—— 换模型 A/B 实验用。
+
+为什么单独一个模块：
+* 生产嵌入走 ollama（bge-m3）。要评估"换模型能不能把召回抬上去"，必须能在
+  不改生产代码路径的前提下换掉**嵌入实现**，同时仍复用 VectorArm 的矩阵缓存、
+  页级去重、重排与融合逻辑 —— 那些是逻辑资产，不该跟着模型一起换。
+* 模型是 int8 ONNX（约 610 MB），本仓库不附带、不下载；路径由
+  ``DSH_KB_EMBED_ONNX_ROOT`` 指定（其下 ``onnx/model_int8.onnx`` + ``tokenizer.json``），
+  依赖库目录由 ``DSH_KB_ONNX_LIB`` 指定（可选，能 import 到即可）。
+
+池化方式（Qwen3-Embedding 官方口径）：
+* 取**最后一个 token** 的 hidden state（不是 CLS，也不是 mean）；
+* L2 归一化；
+* 查询侧加指令前缀 ``Instruct: {task}\nQuery: {query}``，文档侧不加 —— 这是它
+  与非对称检索模型拉开差距的地方，和 bge-m3 的用法不同，别混。
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+import threading
+from pathlib import Path
+
+__all__ = ["available", "embed_texts", "embed_query", "dim", "MODEL_NAME"]
+
+#: Both are unset by default: the ONNX backend is an experiment, not a shipped path.
+_ROOT = os.environ.get("DSH_KB_EMBED_ONNX_ROOT")
+_LIB = os.environ.get("DSH_KB_ONNX_LIB")
+ROOT = Path(_ROOT) if _ROOT else None
+LIB = Path(_LIB) if _LIB else None
+MODEL_NAME = os.environ.get("DSH_KB_EMBED_ONNX_NAME", "qwen3-embedding-0.6b-int8")
+INSTRUCTION = os.environ.get(
+    "DSH_KB_EMBED_INSTRUCTION",
+    "Given a web search query, retrieve relevant passages that answer the query",
+)
+DIM = 1024
+MAX_LEN = int(os.environ.get("DSH_KB_EMBED_MAXLEN", "512"))
+BATCH = int(os.environ.get("DSH_KB_EMBED_BATCH", "8"))
+
+_LOCK = threading.Lock()
+_SESSION = None
+_TOKENIZER = None
+_INPUT_NAMES = ()
+_INPUT_META = ()
+
+
+def _lib_path() -> None:
+    if LIB is not None and LIB.is_dir() and str(LIB) not in sys.path:
+        sys.path.insert(0, str(LIB))
+
+
+def available() -> bool:
+    """模型文件齐全且能 import 依赖时才为 True（不加载模型，不看网络）。"""
+    if ROOT is None:
+        return False
+    need = [ROOT / "onnx" / "model_int8.onnx", ROOT / "tokenizer.json"]
+    if not all(p.is_file() and p.stat().st_size > 1000 for p in need):
+        return False
+    try:
+        _lib_path()
+        import numpy  # noqa: F401
+        import onnxruntime  # noqa: F401
+        import tokenizers  # noqa: F401
+    except Exception:
+        return False
+    return True
+
+
+def _load():
+    global _SESSION, _TOKENIZER, _INPUT_NAMES
+    if _SESSION is not None:
+        return _SESSION, _TOKENIZER
+    with _LOCK:
+        if _SESSION is not None:
+            return _SESSION, _TOKENIZER
+        _lib_path()
+        import onnxruntime as ort
+        from tokenizers import Tokenizer
+
+        opts = ort.SessionOptions()
+        opts.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
+        opts.intra_op_num_threads = int(os.environ.get("DSH_KB_EMBED_THREADS", "4"))
+        opts.inter_op_num_threads = 1
+        # 内存：推理 arena 会一直占着，和别的常驻服务共用一台小机器时很痛。默认
+        # 中间张量池留到进程结束（实测 4 进程各占 3G），关掉后 RSS 降到 ~1.3G。
+        opts.enable_cpu_mem_arena = os.environ.get("DSH_KB_EMBED_ARENA", "0") == "1"
+        sess = ort.InferenceSession(
+            str(ROOT / "onnx" / "model_int8.onnx"), opts, providers=["CPUExecutionProvider"]
+        )
+        tok = Tokenizer.from_file(str(ROOT / "tokenizer.json"))
+        tok.enable_truncation(max_length=MAX_LEN)
+        tok.no_padding()
+        _SESSION, _TOKENIZER = sess, tok
+        global _INPUT_META
+        _INPUT_NAMES = tuple(i.name for i in sess.get_inputs())
+        _INPUT_META = tuple((i.name, list(i.shape or ())) for i in sess.get_inputs())
+        return sess, tok
+
+
+def dim() -> int:
+    return DIM
+
+
+def _postprocess(hidden, mask):
+    """last-token pooling（按 attention_mask 取每条最后一个真实 token）+ L2。"""
+    import numpy as np
+
+    if isinstance(hidden, (list, tuple)):
+        hidden = hidden[0]
+    arr = np.asarray(hidden)
+    if arr.ndim == 3:
+        idx = np.asarray(mask).sum(axis=1) - 1
+        idx = np.clip(idx, 0, arr.shape[1] - 1)
+        arr = arr[np.arange(arr.shape[0]), idx]
+    norms = np.linalg.norm(arr, axis=1, keepdims=True)
+    norms[norms == 0] = 1.0
+    return arr / norms
+
+
+def embed_texts(texts, *, is_query: bool = False, batch_size: int | None = None):
+    """批量嵌入，返回 list[list[float]]（float32 归一化向量）。"""
+    if isinstance(texts, str):
+        texts = [texts]
+    sess, tok = _load()
+    import numpy as np
+
+    bs = batch_size or BATCH
+    prepared = []
+    for t in texts:
+        body = str(t or "").strip()
+        if is_query:
+            body = "Instruct: " + INSTRUCTION + "\nQuery: " + body
+        prepared.append(body or " ")
+    out = []
+    for i in range(0, len(prepared), bs):
+        chunk = prepared[i : i + bs]
+        encs = tok.encode_batch(chunk)
+        maxlen = max(len(e.ids) for e in encs)
+        ids = np.zeros((len(encs), maxlen), dtype="int64")
+        mask = np.zeros((len(encs), maxlen), dtype="int64")
+        for r, e in enumerate(encs):
+            ids[r, : len(e.ids)] = e.ids
+            mask[r, : len(e.ids)] = 1
+        feeds = {
+            "input_ids": ids,
+            "attention_mask": mask,
+            "position_ids": np.tile(np.arange(maxlen, dtype="int64"), (len(encs), 1)),
+        }
+        # 这个 ONNX 导出是「带 KV cache」的解码器图：56 个 past 输入必须显式给空张量
+        # （past_sequence_length=0）。不给就直接 ValueError；给了等价于无缓存的整段前向，
+        # 我们要的正是 last_hidden_state 整段。
+        for name, meta in _INPUT_META:
+            if name in feeds:
+                continue
+            if name.startswith("past_key_values"):
+                shape = [len(encs) if str(s).startswith("batch") else int(s)
+                         if not str(s).startswith("past") else 0
+                         for s in (meta or [])]
+                feeds[name] = np.zeros(shape or (len(encs), 1, 0, 1), dtype="float32")
+        hidden = sess.run(None, feeds)
+        vecs = _postprocess(hidden, mask)
+        out.extend([[float(x) for x in row] for row in vecs])
+    return out
+
+
+def embed_query(text: str):
+    return embed_texts([text], is_query=True)[0]

handrolled/importers/__init__.py

@@ -0,0 +1,68 @@
+"""Adapters that turn somebody else's export into plain records.
+
+Every adapter is a function ``load(path, **options) -> list[Record]`` that reads
+one file or directory and returns plain records.  Nothing here touches the
+network, imports a third-party package, or knows where the knowledge base lives:
+the renderer in :mod:`handrolled.pages` decides that.
+
+Adding an adapter is one module plus one entry in :data:`ADAPTERS`.  The rule for
+accepting one is the same everywhere: if the format cannot be read with the
+standard library and a documented shape, it does not belong here.
+"""
+
+from __future__ import annotations
+
+import importlib
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass(frozen=True)
+class Record:
+    """One unit of knowledge, before records are grouped into a page."""
+
+    content: str
+    #: The note's own title when the source has one (vault pages, basic-memory).
+    title: str = ""
+    id: str = ""
+    tags: tuple[str, ...] = ()
+    type: str = "note"
+    updated_at: str = ""
+    origin: str = ""
+    #: Provenance: a relative path inside the export, or an upstream identifier.
+    source: str = ""
+
+
+#: adapter name -> (what it reads, module name)
+ADAPTERS: dict[str, tuple[str, str]] = {
+    "records": ("the neutral records/v1 JSON every other format converts into", "records"),
+    "jsonl": ("one JSON object per line (content/text/memory/body)", "jsonl"),
+    "mem0": ("a mem0 memory export (JSON array or {\"results\": [...]})", "mem0"),
+    "obsidian": ("an Obsidian vault (Markdown + YAML front matter)", "obsidian"),
+    "notion": ("a Notion \"Markdown & CSV\" export directory", "notion"),
+    "basic-memory": ("basic-memory notes (observations + relations)", "basic_memory"),
+}
+
+
+def names() -> list[str]:
+    return sorted(ADAPTERS)
+
+
+def describe() -> dict[str, str]:
+    return {name: description for name, (description, _) in ADAPTERS.items()}
+
+
+def load(name: str, path: Path, **options) -> list[Record]:
+    """Run one adapter over ``path``.
+
+    Raises ``ValueError`` for an unknown adapter or an input that does not match
+    the shape it claims to read — silently importing nothing is the one outcome
+    worse than failing.
+    """
+    if name not in ADAPTERS:
+        raise ValueError(f"unknown format {name!r}; known formats: {', '.join(names())}")
+    target = Path(path)
+    if not target.exists():
+        raise ValueError(f"no such file or directory: {target}")
+    module = importlib.import_module(f"{__package__}.{ADAPTERS[name][1]}")
+    return module.load(target, **options)

handrolled/importers/basic_memory.py

@@ -0,0 +1,67 @@
+"""basic-memory notes.
+
+basic-memory stores a note as Markdown with ``title``/``type``/``permalink`` in
+the front matter and the knowledge itself as observation bullets:
+
+    - [decision] we chose SQLite over a server
+    - [fact] the corpus is 1.2 GB
+
+The observation is the unit worth indexing, not the page — one note usually
+holds several unrelated statements.  When a page has no observations at all it
+falls back to the whole body, so plain notes still import.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+from . import Record
+from .markdown import iter_markdown, one, page_record, strip_wikilinks, tags_of
+from ..text import parse_frontmatter, split_frontmatter
+
+_OBSERVATION = re.compile(r"^\s*[-*]\s*\[([^\]]+)\]\s*(.+?)\s*$", re.MULTILINE)
+
+
+def load(path: Path, *, wikilinks: str = "strip", default_type: str = "note",
+         **_options) -> list[Record]:
+    out: list[Record] = []
+    for page in iter_markdown(path):
+        try:
+            raw = page.read_text(encoding="utf-8", errors="replace")
+        except OSError:
+            continue
+        front = parse_frontmatter(raw)
+        body, _ = split_frontmatter(raw)
+        if wikilinks == "strip":
+            body = strip_wikilinks(body)
+        try:
+            rel = str(page.relative_to(path))
+        except ValueError:
+            rel = page.name
+
+        observations = _OBSERVATION.findall(body)
+        if not observations:
+            record = page_record(page, path, default_type=default_type,
+                                 wikilinks=wikilinks, origin="basic-memory")
+            if record is not None:
+                out.append(record)
+            continue
+
+        tags = tags_of(front.get("tags"))
+        page_type = one(front.get("type")) or default_type
+        for index, (category, text) in enumerate(observations, 1):
+            text = text.strip()
+            if not text:
+                continue
+            out.append(Record(
+                content=text,
+                title=one(front.get("title")) or page.stem,
+                id=f"{rel}#{index}",
+                tags=tags,
+                type=category.strip() or page_type,
+                updated_at=one(front.get("updated")) or one(front.get("created")),
+                origin="basic-memory",
+                source=rel,
+            ))
+    return out

handrolled/importers/jsonl.py

@@ -0,0 +1,35 @@
+"""JSON Lines — the lowest common denominator of "export everything".
+
+One JSON object per line; blank lines and ``#`` comments are skipped so a file
+can carry a header.  A line that is a bare string becomes the content, which
+covers ``{"content": ...}``-less exports without inventing a second format.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from . import Record
+from .records import coerce
+
+
+def load(path: Path, *, default_type: str = "note", origin: str = "jsonl",
+         **_options) -> list[Record]:
+    out: list[Record] = []
+    text = path.read_text(encoding="utf-8", errors="replace")
+    for number, line in enumerate(text.splitlines(), 1):
+        stripped = line.strip()
+        if not stripped or stripped.startswith("#"):
+            continue
+        try:
+            item = json.loads(stripped)
+        except json.JSONDecodeError as exc:
+            raise ValueError(f"{path}:{number}: not valid JSON ({exc.msg})") from exc
+        if isinstance(item, str):
+            item = {"content": item}
+        record = coerce(item, default_type=default_type, origin=origin,
+                        source=f"{path.name}:{number}")
+        if record is not None:
+            out.append(record)
+    return out

handrolled/importers/markdown.py

@@ -0,0 +1,137 @@
+"""Shared reader for Markdown vaults.
+
+Obsidian, Notion's Markdown export and basic-memory all hand you a directory of
+``.md`` files whose front matter and headings differ only in convention.  One
+reader handles the conventions as options instead of three near-copies:
+
+* front matter is read with :func:`handrolled.text.parse_frontmatter` (the same
+  zero-dependency YAML subset the rest of the project uses);
+* the title is the front matter ``title``, else the first ``#`` heading, else
+  the file name;
+* ``[[wikilinks]]`` are reduced to their label by default — the target indexes plain
+  text, and ``[[target|label]]`` would otherwise be indexed as markup;
+* Notion appends a 32-hex page id to file names; ``strip_hash`` removes it.
+"""
+
+from __future__ import annotations
+
+import re
+from datetime import datetime
+from pathlib import Path
+
+from . import Record
+from ..text import parse_frontmatter, split_frontmatter
+
+#: Directories that hold application state rather than notes.
+SKIP_DIRS = frozenset({
+    ".obsidian", ".trash", ".git", ".stfolder", ".smart-env", ".obsidian-cache",
+    "node_modules", "__pycache__", ".venv", "venv", "dist", "build",
+})
+
+_WIKILINK = re.compile(r"\[\[([^\]|]+)(?:\|([^\]]+))?\]\]")
+_NOTION_ID = re.compile(r"\s+[0-9a-f]{32}$")
+_HEADING = re.compile(r"^\s*#\s+(.+?)\s*$", re.MULTILINE)
+
+
+def iter_markdown(root: Path, *, skip: frozenset[str] = SKIP_DIRS):
+    """Yield ``.md`` files under ``root``, skipping application-state dirs."""
+    root = Path(root)
+    if root.is_file():
+        yield root
+        return
+    for path in sorted(root.rglob("*.md")):
+        rel = path.relative_to(root)
+        if any(part in skip or part.startswith(".") for part in rel.parts[:-1]):
+            continue
+        if path.name.startswith("."):
+            continue
+        yield path
+
+
+def strip_wikilinks(text: str) -> str:
+    """``[[page|label]]`` → ``label``; ``[[page]]`` → ``page``."""
+    return _WIKILINK.sub(lambda m: (m.group(2) or m.group(1)).strip(), text)
+
+
+def one(value) -> str:
+    """Scalar-ish access for a front matter value that may be a list."""
+    if isinstance(value, (list, tuple)):
+        return str(value[0]).strip().strip("'\"") if value else ""
+    if isinstance(value, dict):
+        return ""
+    return str(value).strip().strip("'\"") if value is not None else ""
+
+
+def tags_of(value) -> tuple[str, ...]:
+    if isinstance(value, str):
+        return tuple(part.strip() for part in re.split(r"[,;]", value) if part.strip())
+    if isinstance(value, (list, tuple)):
+        return tuple(str(item).strip() for item in value if str(item).strip())
+    return ()
+
+
+def _stem(path: Path, *, strip_hash: bool) -> str:
+    stem = path.stem
+    return _NOTION_ID.sub("", stem).strip() if strip_hash else stem.strip()
+
+
+def _first_heading(body: str) -> str:
+    match = _HEADING.search(body)
+    return match.group(1).strip() if match else ""
+
+
+def _drop_leading_heading(body: str, title: str) -> str:
+    """Remove the page's own H1 when it just repeats the title we render anyway."""
+    match = _HEADING.match(body)
+    if match and match.group(1).strip().casefold() == title.strip().casefold():
+        return body[match.end():].lstrip("\n")
+    return body
+
+
+def _updated_at(path: Path, front: dict) -> str:
+    for key in ("updated", "updated_at", "created", "created_at", "date"):
+        value = one(front.get(key))
+        if value:
+            return value
+    generated = front.get("generated")
+    if isinstance(generated, dict):
+        value = one(generated.get("at"))
+        if value:
+            return value
+    try:
+        return datetime.fromtimestamp(path.stat().st_mtime).astimezone().isoformat(timespec="seconds")
+    except OSError:
+        return ""
+
+
+def page_record(path: Path, root: Path, *, default_type: str = "note",
+                wikilinks: str = "strip", strip_hash: bool = False,
+                origin: str = "") -> Record | None:
+    """One record per Markdown page; ``None`` when the page has no body."""
+    try:
+        raw = path.read_text(encoding="utf-8", errors="replace")
+    except OSError:
+        return None
+    front = parse_frontmatter(raw)
+    body, _ = split_frontmatter(raw)
+    title = one(front.get("title")) or _first_heading(body) or _stem(path, strip_hash=strip_hash)
+    body = _drop_leading_heading(body.strip(), title)
+    if wikilinks == "strip":
+        body = strip_wikilinks(body)
+    body = body.strip()
+    if not body:
+        return None
+    try:
+        rel = str(path.relative_to(root))
+    except ValueError:
+        rel = path.name
+    return Record(
+        content=body,
+        title=title,
+        id=rel,
+        tags=tags_of(front.get("tags")),
+        type=one(front.get("type")) or default_type,
+        updated_at=_updated_at(path, front),
+        origin=origin or "markdown",
+        source=rel,
+    )