secondbrain-py 0.2.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (273) hide show
  1. brain/__init__.py +0 -0
  2. brain/__main__.py +18 -0
  3. brain/_capture_command.py +445 -0
  4. brain/_compose.py +52 -0
  5. brain/activity.py +206 -0
  6. brain/ask.py +631 -0
  7. brain/audio.py +591 -0
  8. brain/backfill/__init__.py +12 -0
  9. brain/backfill/search_extras.py +141 -0
  10. brain/backfill/source_rows.py +101 -0
  11. brain/bin/__init__.py +1 -0
  12. brain/bin/_launcher.py +107 -0
  13. brain/bin/down.py +8 -0
  14. brain/bin/launchd.py +268 -0
  15. brain/bin/monitor.py +570 -0
  16. brain/bin/rebuild.py +8 -0
  17. brain/bin/status.py +8 -0
  18. brain/bin/up.py +8 -0
  19. brain/brief.py +272 -0
  20. brain/capture.py +49 -0
  21. brain/chat.py +293 -0
  22. brain/cli.py +9760 -0
  23. brain/cli_claude.py +81 -0
  24. brain/cli_connect.py +285 -0
  25. brain/cli_demo.py +266 -0
  26. brain/config.py +1949 -0
  27. brain/connect.py +925 -0
  28. brain/db.py +540 -0
  29. brain/demo/__init__.py +452 -0
  30. brain/demo/corpus/manifest.json +403 -0
  31. brain/demo/embedder.py +74 -0
  32. brain/durations.py +84 -0
  33. brain/edit_session.py +156 -0
  34. brain/editor.py +67 -0
  35. brain/elicit/__init__.py +16 -0
  36. brain/elicit/detectors.py +250 -0
  37. brain/elicit/drafter.py +70 -0
  38. brain/elicit/queue.py +220 -0
  39. brain/elicit/schema.py +48 -0
  40. brain/elicit/session.py +445 -0
  41. brain/embedding_targets.py +54 -0
  42. brain/embeddings.py +424 -0
  43. brain/enrichment.py +808 -0
  44. brain/errors.py +357 -0
  45. brain/eval/__init__.py +129 -0
  46. brain/eval/answer_eval.py +281 -0
  47. brain/eval/baseline.py +265 -0
  48. brain/eval/concept_extraction.py +378 -0
  49. brain/eval/corpus.py +152 -0
  50. brain/eval/errors.py +19 -0
  51. brain/eval/graph_baseline.py +226 -0
  52. brain/eval/graph_retrieval.py +202 -0
  53. brain/eval/graph_runner.py +319 -0
  54. brain/eval/metrics.py +101 -0
  55. brain/eval/runner.py +223 -0
  56. brain/format.py +783 -0
  57. brain/gaps.py +390 -0
  58. brain/graph_rag/__init__.py +94 -0
  59. brain/graph_rag/_retrieval_common.py +113 -0
  60. brain/graph_rag/aggregates.py +303 -0
  61. brain/graph_rag/aliases/__init__.py +583 -0
  62. brain/graph_rag/backends/__init__.py +10 -0
  63. brain/graph_rag/backends/_age_helpers.py +473 -0
  64. brain/graph_rag/backends/age.py +782 -0
  65. brain/graph_rag/backends/base.py +272 -0
  66. brain/graph_rag/build.py +344 -0
  67. brain/graph_rag/communities.py +644 -0
  68. brain/graph_rag/communities_summary.py +437 -0
  69. brain/graph_rag/concepts.py +202 -0
  70. brain/graph_rag/cooccur.py +193 -0
  71. brain/graph_rag/cross_type.py +312 -0
  72. brain/graph_rag/extract.py +885 -0
  73. brain/graph_rag/fuse.py +371 -0
  74. brain/graph_rag/global_.py +412 -0
  75. brain/graph_rag/grouping.py +372 -0
  76. brain/graph_rag/person_resolver.py +167 -0
  77. brain/graph_rag/reconcile.py +792 -0
  78. brain/graph_rag/relational.py +353 -0
  79. brain/graph_rag/retrieve.py +526 -0
  80. brain/graph_rag/router.py +288 -0
  81. brain/graph_rag/schema.py +320 -0
  82. brain/graph_rag/sync.py +237 -0
  83. brain/graph_rag/tenancy.py +43 -0
  84. brain/graph_rag/themes.py +501 -0
  85. brain/graph_rag/weighting.py +202 -0
  86. brain/ingest/__init__.py +1926 -0
  87. brain/ingest/chunker.py +249 -0
  88. brain/ingest/docx.py +40 -0
  89. brain/ingest/gmail.py +621 -0
  90. brain/ingest/markdown.py +37 -0
  91. brain/ingest/pdf.py +61 -0
  92. brain/ingest/stdin.py +22 -0
  93. brain/ingest/sub_tokens.py +91 -0
  94. brain/ingest/text.py +16 -0
  95. brain/interactions.py +205 -0
  96. brain/maintenance.py +355 -0
  97. brain/mcp_server.py +3405 -0
  98. brain/migrations/001_init.sql +43 -0
  99. brain/migrations/002_qwen3_embedding.sql +17 -0
  100. brain/migrations/003_vault_model.sql +41 -0
  101. brain/migrations/004_relax_content_hash_uniqueness.sql +18 -0
  102. brain/migrations/005_derived_links.sql +67 -0
  103. brain/migrations/006_dedup_file_by_source_path.sql +25 -0
  104. brain/migrations/007_email_thread_and_draft.sql +15 -0
  105. brain/migrations/008_gmail_thread_unique.sql +11 -0
  106. brain/migrations/009_chunks_weighted_tsv.sql +28 -0
  107. brain/migrations/010_interactions.sql +30 -0
  108. brain/migrations/011_documents_summary.sql +23 -0
  109. brain/migrations/012_graphrag.sql +171 -0
  110. brain/migrations/013_graphrag_communities.sql +125 -0
  111. brain/migrations/014_graphrag_community_summary_hash.sql +33 -0
  112. brain/migrations/015_interactions_graph_targets.sql +89 -0
  113. brain/migrations/016_index_hygiene.sql +61 -0
  114. brain/migrations/017_elicit.sql +30 -0
  115. brain/migrations/018_review_gap_signal_kinds.sql +40 -0
  116. brain/migrations/019_search_queries.sql +35 -0
  117. brain/migrations/020_link_suggestions.sql +40 -0
  118. brain/migrations/021_timeline_doc_date.sql +34 -0
  119. brain/migrations/022_link_suggestions_undirected.sql +84 -0
  120. brain/migrations/023_search_queries_fts_count.sql +28 -0
  121. brain/quartz_overrides/__init__.py +8 -0
  122. brain/quartz_overrides/quartz/bootstrap-cli.mjs +65 -0
  123. brain/quartz_overrides/quartz/build.ts +568 -0
  124. brain/quartz_overrides/quartz/cli/args.js +152 -0
  125. brain/quartz_overrides/quartz/cli/build_partial_handler.js +544 -0
  126. brain/quartz_overrides/quartz/cli/handlers.js +636 -0
  127. brain/quartz_overrides/quartz/components/CommandPalette.tsx +172 -0
  128. brain/quartz_overrides/quartz/components/Explorer.tsx +198 -0
  129. brain/quartz_overrides/quartz/components/Footer.tsx +27 -0
  130. brain/quartz_overrides/quartz/components/Graph.tsx +468 -0
  131. brain/quartz_overrides/quartz/components/PageTitle.tsx +72 -0
  132. brain/quartz_overrides/quartz/components/RelatedDocs.tsx +38 -0
  133. brain/quartz_overrides/quartz/components/Search.tsx +161 -0
  134. brain/quartz_overrides/quartz/components/SummaryLede.tsx +72 -0
  135. brain/quartz_overrides/quartz/components/index.ts +92 -0
  136. brain/quartz_overrides/quartz/components/pages/TagContent.tsx +272 -0
  137. brain/quartz_overrides/quartz/components/scripts/commandPalette.inline.ts +665 -0
  138. brain/quartz_overrides/quartz/components/scripts/explorer.inline.ts +768 -0
  139. brain/quartz_overrides/quartz/components/scripts/graph.inline.ts +2302 -0
  140. brain/quartz_overrides/quartz/components/scripts/relatedDocs.inline.ts +163 -0
  141. brain/quartz_overrides/quartz/components/scripts/search.inline.ts +1011 -0
  142. brain/quartz_overrides/quartz/plugins/emitters/contentIndex.ts +546 -0
  143. brain/quartz_overrides/quartz/plugins/transformers/codeCopy.ts +94 -0
  144. brain/quartz_overrides/quartz/plugins/transformers/derivedFenceMark.ts +302 -0
  145. brain/quartz_overrides/quartz/plugins/transformers/emailThread.ts +148 -0
  146. brain/quartz_overrides/quartz/plugins/transformers/emptyDoorFilter.ts +213 -0
  147. brain/quartz_overrides/quartz/plugins/transformers/index.ts +114 -0
  148. brain/quartz_overrides/quartz/plugins/transformers/linkKindMark.ts +205 -0
  149. brain/quartz_overrides/quartz/plugins/transformers/linkSourceTag.ts +104 -0
  150. brain/quartz_overrides/quartz/plugins/transformers/relativeDate.ts +100 -0
  151. brain/quartz_overrides/quartz/plugins/transformers/reloadSignal.ts +131 -0
  152. brain/quartz_overrides/quartz/processors/parse.ts +371 -0
  153. brain/quartz_overrides/quartz/processors/parser_cache.ts +78 -0
  154. brain/quartz_overrides/quartz/static/brain-logo-dark.png +0 -0
  155. brain/quartz_overrides/quartz/static/brain-logo-light.png +0 -0
  156. brain/quartz_overrides/quartz/static/codeCopy.js +196 -0
  157. brain/quartz_overrides/quartz/static/emailThread.js +334 -0
  158. brain/quartz_overrides/quartz/static/favicon.ico +0 -0
  159. brain/quartz_overrides/quartz/static/icon.png +0 -0
  160. brain/quartz_overrides/quartz/static/linkSourceTag.js +104 -0
  161. brain/quartz_overrides/quartz/static/relativeDate.js +142 -0
  162. brain/quartz_overrides/quartz/static/reload.js +168 -0
  163. brain/quartz_overrides/quartz/styles/brain/_article.scss +252 -0
  164. brain/quartz_overrides/quartz/styles/brain/_atmosphere.scss +113 -0
  165. brain/quartz_overrides/quartz/styles/brain/_callouts.scss +180 -0
  166. brain/quartz_overrides/quartz/styles/brain/_cmdk.scss +7 -0
  167. brain/quartz_overrides/quartz/styles/brain/_code.scss +208 -0
  168. brain/quartz_overrides/quartz/styles/brain/_command_palette.scss +369 -0
  169. brain/quartz_overrides/quartz/styles/brain/_email_thread.scss +228 -0
  170. brain/quartz_overrides/quartz/styles/brain/_explorer.scss +142 -0
  171. brain/quartz_overrides/quartz/styles/brain/_home.scss +182 -0
  172. brain/quartz_overrides/quartz/styles/brain/_links.scss +322 -0
  173. brain/quartz_overrides/quartz/styles/brain/_marginalia.scss +117 -0
  174. brain/quartz_overrides/quartz/styles/brain/_motion.scss +175 -0
  175. brain/quartz_overrides/quartz/styles/brain/_people_hub.scss +100 -0
  176. brain/quartz_overrides/quartz/styles/brain/_related_docs.scss +137 -0
  177. brain/quartz_overrides/quartz/styles/brain/_search.scss +252 -0
  178. brain/quartz_overrides/quartz/styles/brain/_sidebar.scss +468 -0
  179. brain/quartz_overrides/quartz/styles/brain/_summary_lede.scss +56 -0
  180. brain/quartz_overrides/quartz/styles/brain/_surface.scss +43 -0
  181. brain/quartz_overrides/quartz/styles/brain/_tag_content.scss +118 -0
  182. brain/quartz_overrides/quartz/styles/brain/_tokens.scss +197 -0
  183. brain/quartz_overrides/quartz/styles/brain/_typography.scss +92 -0
  184. brain/quartz_overrides/quartz/styles/custom.scss +89 -0
  185. brain/quartz_overrides/quartz/styles/graph.scss +505 -0
  186. brain/quartz_overrides/quartz/util/ctx.ts +92 -0
  187. brain/quartz_overrides/quartz/util/fastpath_manifest.ts +608 -0
  188. brain/quartz_overrides/quartz/util/path.ts +358 -0
  189. brain/quartz_overrides/quartz/util/sourceIcons.ts +55 -0
  190. brain/quartz_overrides/quartz.config.ts +270 -0
  191. brain/quartz_overrides/quartz.layout.ts +314 -0
  192. brain/queries.py +1188 -0
  193. brain/rank_fusion.py +8 -0
  194. brain/resurface.py +210 -0
  195. brain/review/__init__.py +26 -0
  196. brain/review/emit.py +27 -0
  197. brain/review/queries.py +436 -0
  198. brain/review/render.py +196 -0
  199. brain/review/scans.py +355 -0
  200. brain/review/weekly.py +413 -0
  201. brain/search.py +704 -0
  202. brain/set_similarity.py +15 -0
  203. brain/setup.py +1205 -0
  204. brain/tags.py +56 -0
  205. brain/templates/Caddyfile.j2 +9 -0
  206. brain/templates/__init__.py +1 -0
  207. brain/templates/bin/__init__.py +1 -0
  208. brain/templates/bin/_brain-brief-fg.sh +25 -0
  209. brain/templates/bin/_brain-build-fg.sh +53 -0
  210. brain/templates/bin/_brain-watcher-fg.sh +65 -0
  211. brain/templates/bin/brain-down.sh +89 -0
  212. brain/templates/bin/brain-status.sh +83 -0
  213. brain/templates/bin/brain-up.sh +221 -0
  214. brain/templates/docker/age/Dockerfile +79 -0
  215. brain/templates/docker-compose.stock.yml.j2 +26 -0
  216. brain/templates/docker-compose.yml.j2 +34 -0
  217. brain/templates/env.example +190 -0
  218. brain/templates/launchd/__init__.py +1 -0
  219. brain/templates/launchd/com.brain.brief.plist.j2 +45 -0
  220. brain/templates/launchd/com.brain.build.plist.j2 +46 -0
  221. brain/templates/launchd/com.brain.watcher.plist.j2 +46 -0
  222. brain/templates/skill/SKILL.md +63 -0
  223. brain/templates/skill/__init__.py +1 -0
  224. brain/timeline.py +834 -0
  225. brain/todo.py +124 -0
  226. brain/uninstall.py +185 -0
  227. brain/vault/__init__.py +115 -0
  228. brain/vault/_atomic.py +25 -0
  229. brain/vault/daily_index.py +228 -0
  230. brain/vault/derived_links/__init__.py +50 -0
  231. brain/vault/derived_links/directory.py +683 -0
  232. brain/vault/derived_links/fence.py +408 -0
  233. brain/vault/derived_links/gws.py +64 -0
  234. brain/vault/derived_links/participants.py +143 -0
  235. brain/vault/derived_links/pass_runner.py +362 -0
  236. brain/vault/derived_links/rules.py +137 -0
  237. brain/vault/export.py +683 -0
  238. brain/vault/frontmatter.py +165 -0
  239. brain/vault/graph.py +620 -0
  240. brain/vault/graph_format.py +388 -0
  241. brain/vault/link_rewrite.py +235 -0
  242. brain/vault/links.py +260 -0
  243. brain/vault/note_builder.py +211 -0
  244. brain/vault/paths.py +55 -0
  245. brain/vault/quartz_overlay.py +236 -0
  246. brain/vault/rename.py +591 -0
  247. brain/vault/resolver.py +304 -0
  248. brain/vault/slug.py +127 -0
  249. brain/vault/sync.py +1513 -0
  250. brain/vault/sync_summaries.py +264 -0
  251. brain/vault/templates.py +145 -0
  252. brain/vault/watch.py +1052 -0
  253. brain/wiki/__init__.py +6 -0
  254. brain/wiki/_github_slugger.py +76 -0
  255. brain/wiki/_person_name.py +314 -0
  256. brain/wiki/build_homepage.py +541 -0
  257. brain/wiki/build_partial.py +273 -0
  258. brain/wiki/build_people.py +934 -0
  259. brain/wiki/build_related.py +758 -0
  260. brain/wiki/build_swap.py +585 -0
  261. brain/wiki/build_watcher.py +975 -0
  262. brain/wiki/edit_classifier.py +215 -0
  263. brain/wiki/errors.py +10 -0
  264. brain/wiki/fastpath_manifest.py +475 -0
  265. brain/wiki/fastpath_state.py +174 -0
  266. brain/wiki/install.py +296 -0
  267. brain/wiki/slug.py +111 -0
  268. secondbrain_py-0.2.1.dist-info/METADATA +195 -0
  269. secondbrain_py-0.2.1.dist-info/RECORD +273 -0
  270. secondbrain_py-0.2.1.dist-info/WHEEL +5 -0
  271. secondbrain_py-0.2.1.dist-info/entry_points.txt +11 -0
  272. secondbrain_py-0.2.1.dist-info/licenses/LICENSE +21 -0
  273. secondbrain_py-0.2.1.dist-info/top_level.txt +1 -0
@@ -0,0 +1,608 @@
1
+ // Brain wiki — fast-path manifest: per-slug structural fingerprint + manifest I/O.
2
+ //
3
+ // SYNC NOTE: This file mirrors tests/wiki/fixtures/fingerprint_parity_runner.mjs
4
+ // (and vice-versa). When you change the canonical-blob shape, FINGERPRINT_VERSION,
5
+ // or any _STRUCTURAL_FIELDS / _ARRAY_FIELDS / _BOOL_FIELDS / _DATE_FIELDS /
6
+ // _STRIP_ASCII_SET constant, edit BOTH files. The static test
7
+ // test_runner_and_ts_declare_identical_constants guards against accidental drift.
8
+ //
9
+ // This file is a TEMPLATE. It is installed at
10
+ // `<vault>/.quartz/quartz/util/fastpath_manifest.ts` by
11
+ // `brain vault render --overlay`.
12
+ //
13
+ // Canonical-blob format: docs/specs/2026-05-09-fastpath-fingerprint.md
14
+ //
15
+ // The Python counterpart (src/brain/wiki/fastpath_manifest.py) MUST produce
16
+ // byte-identical canonical blobs for the same inputs. The parity test at
17
+ // tests/wiki/test_fastpath_fingerprint_parity.py enforces this contract.
18
+ //
19
+ // Usage by T2 (full-build hook):
20
+ // import { computeFingerprint, writeManifest } from "./util/fastpath_manifest"
21
+ // const slugEntries: Record<string, SlugEntry> = {}
22
+ // for (const pc of filteredContent) {
23
+ // const [, vfile] = pc
24
+ // const slug = vfile.data.slug!
25
+ // // relativePath is vault-relative (e.g. "notes/my-note.md"), NOT the full disk path.
26
+ // // outputPath may differ from slug+".html" for folder-index files.
27
+ // const sourcePath = String(vfile.data.relativePath ?? "")
28
+ // const outputPath = slug + ".html" // T2: may be slug+"/index.html" for folder indexes
29
+ // slugEntries[slug] = {
30
+ // fingerprint: computeFingerprint(pc, { sourcePath, outputPath }),
31
+ // output_path: outputPath,
32
+ // source_path: sourcePath,
33
+ // }
34
+ // }
35
+ // writeManifest(fastpathDir, {
36
+ // version: FINGERPRINT_VERSION,
37
+ // parent_build_id: process.env.QUARTZ_PARENT_BUILD_ID ?? "",
38
+ // built_at_ms: Date.now(),
39
+ // slugs: slugEntries,
40
+ // })
41
+
42
+ import { createHash, randomUUID } from "node:crypto"
43
+ import { mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"
44
+ import { dirname, join } from "node:path"
45
+ import type { ProcessedContent } from "../plugins/vfile"
46
+
47
+ export const FINGERPRINT_VERSION: number = 1
48
+
49
+ // ---------------------------------------------------------------------------
50
+ // Public types
51
+ // ---------------------------------------------------------------------------
52
+
53
+ export interface SlugEntry {
54
+ /** sha256 hex fingerprint of the canonical blob. */
55
+ fingerprint: string
56
+ /** HTML output path relative to the build directory (e.g. "my-note.html"). */
57
+ output_path: string
58
+ /** Markdown source path relative to the vault root (e.g. "notes/my-note.md").
59
+ * Callers MUST supply this explicitly — use ``vfile.data.relativePath``, NOT
60
+ * ``vfile.data.filePath`` (which is the full absolute disk path). */
61
+ source_path: string
62
+ }
63
+
64
+ export interface Manifest {
65
+ /** Must equal FINGERPRINT_VERSION at write time. */
66
+ version: number
67
+ /** Build-id of the full build that wrote this manifest (matches current/.build-id). */
68
+ parent_build_id: string
69
+ /** Unix epoch milliseconds when this manifest was written. */
70
+ built_at_ms: number
71
+ /** Per-slug fingerprint entries, keyed by Quartz FullSlug. */
72
+ slugs: Record<string, SlugEntry>
73
+ }
74
+
75
+ // ---------------------------------------------------------------------------
76
+ // Structural frontmatter field order (must match Python _STRUCTURAL_FIELD_ORDER).
77
+ // ---------------------------------------------------------------------------
78
+
79
+ const _STRUCTURAL_FIELDS = [
80
+ "title", "draft", "publish", "tags", "aliases", "permalink", "slug",
81
+ "lang", "cssclasses", "socialImage", "enableToc", "comments", "kind",
82
+ "description", "socialDescription", "date", "created", "modified",
83
+ "updated", "published",
84
+ ] as const
85
+
86
+ const _ARRAY_FIELDS = new Set(["tags", "aliases", "cssclasses"])
87
+ const _BOOL_FIELDS = new Set(["draft", "publish", "enableToc", "comments"])
88
+ const _DATE_FIELDS = new Set(["date", "created", "modified", "updated", "published"])
89
+
90
+ // ---------------------------------------------------------------------------
91
+ // Canonical blob encoding helpers
92
+ // ---------------------------------------------------------------------------
93
+
94
+ function _u32be(n: number): Buffer {
95
+ const buf = Buffer.allocUnsafe(4)
96
+ buf.writeUInt32BE(n, 0)
97
+ return buf
98
+ }
99
+
100
+ function _encodeSection(s: string): Buffer {
101
+ const encoded = Buffer.from(s, "utf8")
102
+ return Buffer.concat([_u32be(encoded.length), encoded])
103
+ }
104
+
105
+ // ---------------------------------------------------------------------------
106
+ // github-slugger reimplementation (matches npm github-slugger v2.x exactly).
107
+ // Inlined so this file has no runtime import that would break when loaded
108
+ // outside the Quartz workspace (e.g. via tsx in the parity test runner).
109
+ // The Quartz overlay version (with ProcessedContent import) uses the same
110
+ // algorithm; the parity runner script also inlines this same code.
111
+ // ---------------------------------------------------------------------------
112
+
113
+ const _STRIP_ASCII_SET = new Set([
114
+ "\\", "'", '"', "!", "#", "$", "%", "&", "(", ")", "*", "+",
115
+ ",", ".", "/", ":", ";", "<", "=", ">", "?", "@", "[", "]",
116
+ "^", "`", "{", "|", "}", "~",
117
+ ])
118
+
119
+ function _slugNormalize(text: string): string {
120
+ // NFC + lowercase
121
+ text = text.normalize("NFC").toLowerCase().trim()
122
+ const result: string[] = []
123
+ for (const ch of text) {
124
+ const cp = ch.codePointAt(0)!
125
+ if (_STRIP_ASCII_SET.has(ch)) continue
126
+ if (cp >= 0x2000 && cp <= 0x206f) continue
127
+ if (cp >= 0x2e00 && cp <= 0x2e7f) continue
128
+ result.push(ch)
129
+ }
130
+ return result.join("").replace(/\s+/g, "-")
131
+ }
132
+
133
+ class _Slugger {
134
+ private seen = new Map<string, number>()
135
+
136
+ slug(text: string): string {
137
+ const base = _slugNormalize(text)
138
+ if (!this.seen.has(base)) {
139
+ this.seen.set(base, 0)
140
+ return base
141
+ }
142
+ const count = (this.seen.get(base)! + 1)
143
+ this.seen.set(base, count)
144
+ return `${base}-${count}`
145
+ }
146
+ }
147
+
148
+ // ---------------------------------------------------------------------------
149
+ // YAML frontmatter tag extraction from raw source
150
+ // (needed to get YAML-only tags for SECTION_FRONTMATTER, before OFM merge).
151
+ // ---------------------------------------------------------------------------
152
+
153
+ /**
154
+ * Parse YAML-only tags from raw file source.
155
+ * Returns null when the ``tags:`` key is ABSENT (parity: Python returns null too).
156
+ * Returns [] when tags: is present but empty.
157
+ * Returns [tag, ...] when tags are declared.
158
+ */
159
+ function _parseYamlTags(fileSource: string): string[] | null {
160
+ // Extract the YAML block between --- delimiters.
161
+ const blockMatch = /^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/.exec(fileSource)
162
+ if (!blockMatch) return null // no frontmatter → tags key absent
163
+
164
+ const yaml = blockMatch[1]
165
+
166
+ // Case 1: inline array — tags: [a, b, c] or tags: []
167
+ const inlineArr = /^tags:\s*\[([^\]]*)\]/m.exec(yaml)
168
+ if (inlineArr) {
169
+ return inlineArr[1]
170
+ .split(",")
171
+ .map((s) => s.trim().replace(/^['"]|['"]$/g, ""))
172
+ .filter(Boolean)
173
+ }
174
+
175
+ // Case 2: block list —
176
+ // tags:
177
+ // - item
178
+ const blockListMatch = /^tags:\s*$/m.exec(yaml)
179
+ if (blockListMatch) {
180
+ const after = yaml.slice(blockListMatch.index + blockListMatch[0].length)
181
+ const items: string[] = []
182
+ for (const line of after.split("\n")) {
183
+ const m = /^[ \t]*-\s*(.+)/.exec(line)
184
+ if (m) {
185
+ items.push(m[1].trim().replace(/^['"]|['"]$/g, ""))
186
+ } else if (line.trim() !== "" && !/^[ \t]/.test(line)) {
187
+ break
188
+ }
189
+ }
190
+ return items // may be [] if block was empty
191
+ }
192
+
193
+ // Case 3: scalar — tags: single-tag
194
+ const scalar = /^tags:\s+([^\[\r\n].+)/m.exec(yaml)
195
+ if (scalar) {
196
+ const val = scalar[1].trim().replace(/^['"]|['"]$/g, "")
197
+ return val ? [val] : []
198
+ }
199
+
200
+ // tags: key not found in YAML block
201
+ return null
202
+ }
203
+
204
+ // ---------------------------------------------------------------------------
205
+ // Body extraction: strip YAML frontmatter from raw source markdown.
206
+ // Shared by computeFingerprint (ProcessedContent path) and
207
+ // computeFingerprintFromSource (raw-source path) so both see identical body text.
208
+ // ---------------------------------------------------------------------------
209
+
210
+ function _extractBody(rawSource: string): string {
211
+ const fmMatch = /^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/.exec(rawSource)
212
+ return fmMatch ? rawSource.slice(fmMatch[0].length) : rawSource
213
+ }
214
+
215
+ // ---------------------------------------------------------------------------
216
+ // Date normalization — Date objects from js-yaml → ISO string.
217
+ // ---------------------------------------------------------------------------
218
+
219
+ function _normalizeDateVal(val: unknown): string | null {
220
+ if (val === null || val === undefined) return null
221
+ if (val instanceof Date) {
222
+ // Normalize to Python datetime.isoformat()-compatible format (no timezone suffix,
223
+ // no milliseconds) so TS and Python blobs are byte-identical for the same YAML:
224
+ // "2024-03-15T00:00:00.000Z" → "2024-03-15" (midnight UTC = date-only)
225
+ // "2024-03-15T12:00:00.000Z" → "2024-03-15T12:00:00" (non-midnight datetime)
226
+ // Matches Python _normalize_date_val() output for datetime.date / datetime.datetime.
227
+ const noZone = val.toISOString().replace(/\.000Z$/, "").replace(/Z$/, "")
228
+ return noZone.endsWith("T00:00:00") ? noZone.slice(0, 10) : noZone
229
+ }
230
+ const s = String(val)
231
+ // String dates from _parseMinimalYaml: normalize midnight datetimes to date-only to
232
+ // match Python pyyaml → _normalize_date_val which truncates midnight datetimes.
233
+ // Python: yaml.safe_load("2024-01-10T00:00:00") → datetime.datetime(2024,1,10,0,0,0)
234
+ // → _normalize_date_val → "2024-01-10" (midnight truncated)
235
+ // TS: _parseMinimalYaml → "2024-01-10T00:00:00" (string) → must also truncate.
236
+ if (/^\d{4}-\d{2}-\d{2}T00:00:00$/.test(s)) return s.slice(0, 10)
237
+ return s
238
+ }
239
+
240
+ // ---------------------------------------------------------------------------
241
+ // Frontmatter JSON blob builder (structural fields, deterministic key order).
242
+ // ---------------------------------------------------------------------------
243
+
244
+ function _buildFrontmatterJson(
245
+ fm: Record<string, unknown>,
246
+ yamlTags: string[] | null,
247
+ ): string {
248
+ const obj: Record<string, unknown> = {}
249
+ for (const key of _STRUCTURAL_FIELDS) {
250
+ // For tags: use YAML-only (pre-OFM) tags. null = key absent in YAML.
251
+ const rawVal = key === "tags" ? yamlTags : fm[key]
252
+
253
+ if (_ARRAY_FIELDS.has(key)) {
254
+ if (rawVal === undefined || rawVal === null) {
255
+ obj[key] = null
256
+ } else {
257
+ const arr = Array.isArray(rawVal)
258
+ ? (rawVal as unknown[]).map(String).filter(Boolean)
259
+ : typeof rawVal === "string"
260
+ ? [rawVal]
261
+ : []
262
+ obj[key] = arr.slice().sort()
263
+ }
264
+ } else if (_BOOL_FIELDS.has(key)) {
265
+ obj[key] = rawVal === undefined || rawVal === null ? null : Boolean(rawVal)
266
+ } else if (_DATE_FIELDS.has(key)) {
267
+ obj[key] = _normalizeDateVal(rawVal)
268
+ } else {
269
+ obj[key] = rawVal === undefined || rawVal === null ? null : String(rawVal)
270
+ }
271
+ }
272
+ return JSON.stringify(obj)
273
+ }
274
+
275
+ // ---------------------------------------------------------------------------
276
+ // Body parsing: wikilinks, transclusions, block-refs, headings
277
+ // ---------------------------------------------------------------------------
278
+
279
+ /** Non-transclusion wikilinks: [[target]] or [[target|alias]]. */
280
+ const _WIKILINK_RE = /(?<!!)\[\[([^\[\]|]+?)(?:\|[^\[\]]*)?\]\]/g
281
+
282
+ /** Transclusions: ![[target]] including anchors. */
283
+ const _TRANSCLUSION_RE = /!\[\[([^\[\]]+?)\]\]/g
284
+
285
+ /**
286
+ * Block-ref IDs defined in body: ^blockid at end of line.
287
+ * Negative lookbehind for [ and # to avoid matching inside wikilinks/anchors.
288
+ */
289
+ const _BLOCKREF_DEF_RE = /(?<![[\#])\^([A-Za-z0-9][A-Za-z0-9-]*)(?=[ \t]*$)/gm
290
+
291
+ /** ATX headings: # text or ## text (with optional trailing ##). */
292
+ const _HEADING_RE = /^#{1,6}[ \t]+(.+?)(?:[ \t]+#+)?$/gm
293
+
294
+ /** Inline body tags: #tagname preceded by whitespace or start of line. */
295
+ const _INLINE_TAG_RE = /(?:^|(?<=\s))#([A-Za-zÀ-ɏͰ-Ͽ一-龥_][^\s#,;!@$%^&*()\[\]{}'\"<>?\/\\|`]*)/gm
296
+
297
+ function _extractInlineTags(body: string): string[] {
298
+ const tags: string[] = []
299
+ let m: RegExpExecArray | null
300
+ const re = new RegExp(_INLINE_TAG_RE.source, _INLINE_TAG_RE.flags)
301
+ while ((m = re.exec(body)) !== null) {
302
+ tags.push(m[1])
303
+ }
304
+ return tags
305
+ }
306
+
307
+ function _extractWikilinks(body: string): string[] {
308
+ const targets: string[] = []
309
+ let m: RegExpExecArray | null
310
+ const re = new RegExp(_WIKILINK_RE.source, _WIKILINK_RE.flags)
311
+ while ((m = re.exec(body)) !== null) {
312
+ const t = m[1].trim()
313
+ if (t) targets.push(t)
314
+ }
315
+ return targets
316
+ }
317
+
318
+ function _extractTransclusions(body: string): string[] {
319
+ const targets: string[] = []
320
+ let m: RegExpExecArray | null
321
+ const re = new RegExp(_TRANSCLUSION_RE.source, _TRANSCLUSION_RE.flags)
322
+ while ((m = re.exec(body)) !== null) {
323
+ const t = m[1].trim()
324
+ if (t) targets.push(t)
325
+ }
326
+ return targets
327
+ }
328
+
329
+ function _extractBlockRefs(body: string): string[] {
330
+ const refs: string[] = []
331
+ let m: RegExpExecArray | null
332
+ const re = new RegExp(_BLOCKREF_DEF_RE.source, _BLOCKREF_DEF_RE.flags)
333
+ while ((m = re.exec(body)) !== null) {
334
+ refs.push(m[1])
335
+ }
336
+ return refs
337
+ }
338
+
339
+ function _extractHeadingAnchors(body: string): string[] {
340
+ const slugger = new _Slugger()
341
+ const anchors: string[] = []
342
+ let m: RegExpExecArray | null
343
+ const re = new RegExp(_HEADING_RE.source, _HEADING_RE.flags)
344
+ while ((m = re.exec(body)) !== null) {
345
+ anchors.push(slugger.slug(m[1].trim()))
346
+ }
347
+ return anchors
348
+ }
349
+
350
+ // ---------------------------------------------------------------------------
351
+ // Public API
352
+ // ---------------------------------------------------------------------------
353
+
354
+ /**
355
+ * Compute the structural fingerprint for one ``ProcessedContent`` entry.
356
+ *
357
+ * Called at full-build time over each entry in ``filteredContent``.
358
+ * Reads the pre-OFM raw source from ``vfile.data.rawSource`` — a snapshot
359
+ * set by ``processors/parse.ts`` AFTER text transforms but BEFORE
360
+ * ``processor.run`` (where Quartz's OFM transformer mutates ``vfile.value``,
361
+ * e.g. appending ``|^blockId`` display aliases to transclusion links like
362
+ * ``![[target#^block]]`` → ``![[target#^block|^block]]``).
363
+ *
364
+ * Throws if ``vfile.data.rawSource`` is absent — ``parse.ts`` MUST set the
365
+ * snapshot before any transformer runs.
366
+ *
367
+ * MUST produce byte-identical blobs as the Python counterpart for the same
368
+ * source file. Verified by ``tests/wiki/test_fastpath_fingerprint_parity.py``.
369
+ */
370
+ export function computeFingerprint(
371
+ processedContent: ProcessedContent,
372
+ paths: { sourcePath: string; outputPath: string },
373
+ ): string {
374
+ const [, vfile] = processedContent
375
+ const slug = vfile.data.slug!
376
+ const { sourcePath, outputPath } = paths
377
+
378
+ // Use the pre-OFM snapshot stored by processors/parse.ts.
379
+ // parse.ts sets vfile.data.rawSource = rawBytes.toString("utf8") after readFile
380
+ // but before processor.run, ensuring we see the unmodified markdown text.
381
+ const rawSource = (vfile.data as Record<string, unknown>)["rawSource"]
382
+ if (rawSource === undefined || rawSource === null) {
383
+ throw new Error(
384
+ "vfile.data.rawSource missing — processors/parse.ts must snapshot raw source " +
385
+ "before processor.run (OFM transformer); cannot compute fingerprint for slug: " +
386
+ String(slug),
387
+ )
388
+ }
389
+ const fileSource = String(rawSource)
390
+
391
+ // Parse structural frontmatter from RAW source, NOT from vfile.data.frontmatter.
392
+ //
393
+ // vfile.data.frontmatter is mutated by Quartz transformers:
394
+ // - title is set to file.stem for files that have no explicit title field
395
+ // - date fields are parsed by js-yaml as Date objects using the host machine's
396
+ // local timezone offset (e.g. PST: "2024-01-10T00:00:00" → T08:00:00Z)
397
+ //
398
+ // Using _parseMinimalYaml on the raw source makes computeFingerprint
399
+ // byte-identical to computeFingerprintFromSource and the Python counterpart,
400
+ // both of which parse from raw text rather than the post-transformer object.
401
+ const fmMatch = /^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/.exec(fileSource)
402
+ const fm: Record<string, unknown> = fmMatch ? _parseMinimalYaml(fmMatch[1]) : {}
403
+
404
+ // YAML-only tags (before inline-tag merge) — parse from raw source.
405
+ // null means tags: key is absent from YAML frontmatter.
406
+ const yamlTags = _parseYamlTags(fileSource)
407
+
408
+ // Body: raw markdown minus frontmatter.
409
+ // Sourced from vfile.data.rawSource (parse-time snapshot) — NOT from vfile.value
410
+ // (OFM-mutated) or vfile.data.text (rendered plain text that strips OFM syntax).
411
+ const body = _extractBody(fileSource)
412
+
413
+ // SECTION_TAGS: merged YAML tags + inline body tags extracted from raw source.
414
+ // We do NOT use vfile.data.frontmatter.tags (post-OFM-transformer) because
415
+ // Quartz's OFM may classify inline content differently from our regex (e.g., it
416
+ // processes wikilink anchors and block-refs in a way that can pollute the tags
417
+ // array). Using _extractInlineTags here makes production computeFingerprint
418
+ // byte-identical to computeFingerprintFromSource and the Python counterpart.
419
+ const inlineTags = _extractInlineTags(body)
420
+ const mergedTagsRaw = Array.from(new Set([...(yamlTags ?? []), ...inlineTags]))
421
+
422
+ return _computeFingerprintFromParts(
423
+ slug, sourcePath, outputPath, fm, yamlTags, mergedTagsRaw, body,
424
+ )
425
+ }
426
+
427
+ /**
428
+ * Compute fingerprint from raw source text (no ProcessedContent needed).
429
+ *
430
+ * Used by the parity test runner (no Quartz runtime available). Accepts the
431
+ * raw file source (frontmatter + body) plus the pre-computed slug and paths.
432
+ */
433
+ export function computeFingerprintFromSource(params: {
434
+ slug: string
435
+ source_path: string
436
+ output_path: string
437
+ source_text: string
438
+ }): string {
439
+ const { slug, source_path, output_path, source_text } = params
440
+
441
+ // Parse frontmatter from raw source.
442
+ const fmMatch = /^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/.exec(source_text)
443
+ let fm: Record<string, unknown> = {}
444
+ if (fmMatch) {
445
+ // Minimal YAML parser for structural fields — covers string, bool, array types.
446
+ fm = _parseMinimalYaml(fmMatch[1])
447
+ }
448
+ const body = _extractBody(source_text)
449
+
450
+ const yamlTags = _parseYamlTags(source_text) // null | string[]
451
+ const inlineTags = _extractInlineTags(body)
452
+ // yamlTags null = key absent → treat as [] for merge
453
+ const mergedTagsRaw = Array.from(new Set([...(yamlTags ?? []), ...inlineTags]))
454
+
455
+ return _computeFingerprintFromParts(
456
+ slug, source_path, output_path, fm, yamlTags, mergedTagsRaw, body,
457
+ )
458
+ }
459
+
460
+ // ---------------------------------------------------------------------------
461
+ // Internal shared computation
462
+ // ---------------------------------------------------------------------------
463
+
464
+ function _computeFingerprintFromParts(
465
+ slug: string,
466
+ sourcePath: string,
467
+ outputPath: string,
468
+ fm: Record<string, unknown>,
469
+ yamlTags: string[] | null,
470
+ mergedTagsRaw: string[],
471
+ body: string,
472
+ ): string {
473
+ const fmJson = _buildFrontmatterJson(fm, yamlTags)
474
+
475
+ const mergedTags = Array.from(new Set(mergedTagsRaw)).sort()
476
+ const tagsStr = mergedTags.join("\n")
477
+
478
+ const wikilinks = Array.from(new Set(_extractWikilinks(body))).sort()
479
+ const wikilinksStr = wikilinks.join("\n")
480
+
481
+ const transclusions = Array.from(new Set(_extractTransclusions(body))).sort()
482
+ const transclusionsStr = transclusions.join("\n")
483
+
484
+ const blockRefs = Array.from(new Set(_extractBlockRefs(body))).sort()
485
+ const blockRefsStr = blockRefs.join("\n")
486
+
487
+ const headingsStr = _extractHeadingAnchors(body).join("\n")
488
+
489
+ const blob = Buffer.concat([
490
+ _u32be(FINGERPRINT_VERSION),
491
+ _encodeSection(slug),
492
+ _encodeSection(sourcePath),
493
+ _encodeSection(outputPath),
494
+ _encodeSection(fmJson),
495
+ _encodeSection(tagsStr),
496
+ _encodeSection(wikilinksStr),
497
+ _encodeSection(transclusionsStr),
498
+ _encodeSection(blockRefsStr),
499
+ _encodeSection(headingsStr),
500
+ ])
501
+ return createHash("sha256").update(blob).digest("hex")
502
+ }
503
+
504
+ // ---------------------------------------------------------------------------
505
+ // Minimal YAML parser for computeFingerprintFromSource (structural fields only)
506
+ // ---------------------------------------------------------------------------
507
+
508
+ function _parseMinimalYaml(yamlText: string): Record<string, unknown> {
509
+ const result: Record<string, unknown> = {}
510
+ const lines = yamlText.split("\n")
511
+ let i = 0
512
+ while (i < lines.length) {
513
+ const line = lines[i]
514
+ const keyMatch = /^([a-zA-Z_][a-zA-Z0-9_-]*):\s*(.*)$/.exec(line)
515
+ if (!keyMatch) { i++; continue }
516
+ const key = keyMatch[1]
517
+ const rest = keyMatch[2].trim()
518
+
519
+ if (rest === "") {
520
+ // Could be a block list: next lines are " - item"
521
+ const items: string[] = []
522
+ let j = i + 1
523
+ while (j < lines.length) {
524
+ const itemMatch = /^[ \t]*-\s*(.+)/.exec(lines[j])
525
+ if (itemMatch) {
526
+ items.push(itemMatch[1].trim().replace(/^['"]|['"]$/g, ""))
527
+ j++
528
+ } else if (lines[j].trim() === "") {
529
+ j++
530
+ } else {
531
+ break
532
+ }
533
+ }
534
+ if (items.length > 0) {
535
+ result[key] = items
536
+ i = j
537
+ continue
538
+ }
539
+ result[key] = null
540
+ } else if (rest.startsWith("[")) {
541
+ // Inline array: [a, b, c]
542
+ const arrMatch = /^\[([^\]]*)\]/.exec(rest)
543
+ if (arrMatch) {
544
+ result[key] = arrMatch[1]
545
+ .split(",")
546
+ .map((s) => s.trim().replace(/^['"]|['"]$/g, ""))
547
+ .filter(Boolean)
548
+ }
549
+ } else if (rest === "true" || rest === "True" || rest === "TRUE") {
550
+ result[key] = true
551
+ } else if (rest === "false" || rest === "False" || rest === "FALSE") {
552
+ result[key] = false
553
+ } else if (rest === "null" || rest === "~" || rest === "Null") {
554
+ result[key] = null
555
+ } else {
556
+ // String or number — strip quotes, keep as string for structural fields.
557
+ result[key] = rest.replace(/^['"]|['"]$/g, "")
558
+ }
559
+ i++
560
+ }
561
+ return result
562
+ }
563
+
564
+ // ---------------------------------------------------------------------------
565
+ // Manifest I/O
566
+ // ---------------------------------------------------------------------------
567
+
568
+ const _MANIFEST_FILENAME = "manifest.json"
569
+
570
+ /**
571
+ * Atomically write ``data`` as JSON to ``<dir>/<filename>``.
572
+ *
573
+ * Atomic strategy: write to ``<dir>/<filename>.<pid>.<uuid>.tmp``, then
574
+ * ``renameSync`` to the final path. On POSIX, ``rename(2)`` is atomic; on
575
+ * Windows it replaces atomically on NTFS.
576
+ *
577
+ * Shared by ``writeManifest`` and the contentmap writer in ``build.ts`` to
578
+ * avoid duplicating the tmp-write + rename strategy.
579
+ */
580
+ export function _atomicWriteJson(dir: string, filename: string, data: unknown): void {
581
+ mkdirSync(dir, { recursive: true })
582
+ const final = join(dir, filename)
583
+ const tmp = join(dir, `${filename}.${process.pid}.${randomUUID()}.tmp`)
584
+ writeFileSync(tmp, JSON.stringify(data), "utf8")
585
+ renameSync(tmp, final)
586
+ }
587
+
588
+ /**
589
+ * Atomically write ``manifest`` to ``<dir>/manifest.json``.
590
+ *
591
+ * Delegates to ``_atomicWriteJson`` for the shared write-tmp + rename strategy.
592
+ */
593
+ export function writeManifest(dir: string, manifest: Manifest): void {
594
+ _atomicWriteJson(dir, _MANIFEST_FILENAME, manifest)
595
+ }
596
+
597
+ /**
598
+ * Read and JSON-parse ``<dir>/manifest.json``.
599
+ *
600
+ * Throws on file-not-found, JSON parse error, or type mismatch — callers
601
+ * should catch and treat any error as «manifest unavailable → full build».
602
+ */
603
+ export function readManifest(dir: string): Manifest {
604
+ const path = join(dir, _MANIFEST_FILENAME)
605
+ const raw = readFileSync(path, "utf8")
606
+ const data = JSON.parse(raw) as Manifest
607
+ return data
608
+ }