secondbrain-py 0.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- brain/__init__.py +0 -0
- brain/__main__.py +18 -0
- brain/_capture_command.py +445 -0
- brain/_compose.py +52 -0
- brain/activity.py +206 -0
- brain/ask.py +631 -0
- brain/audio.py +591 -0
- brain/backfill/__init__.py +12 -0
- brain/backfill/search_extras.py +141 -0
- brain/backfill/source_rows.py +101 -0
- brain/bin/__init__.py +1 -0
- brain/bin/_launcher.py +107 -0
- brain/bin/down.py +8 -0
- brain/bin/launchd.py +268 -0
- brain/bin/monitor.py +570 -0
- brain/bin/rebuild.py +8 -0
- brain/bin/status.py +8 -0
- brain/bin/up.py +8 -0
- brain/brief.py +272 -0
- brain/capture.py +49 -0
- brain/chat.py +293 -0
- brain/cli.py +9760 -0
- brain/cli_claude.py +81 -0
- brain/cli_connect.py +285 -0
- brain/cli_demo.py +266 -0
- brain/config.py +1949 -0
- brain/connect.py +925 -0
- brain/db.py +540 -0
- brain/demo/__init__.py +452 -0
- brain/demo/corpus/manifest.json +403 -0
- brain/demo/embedder.py +74 -0
- brain/durations.py +84 -0
- brain/edit_session.py +156 -0
- brain/editor.py +67 -0
- brain/elicit/__init__.py +16 -0
- brain/elicit/detectors.py +250 -0
- brain/elicit/drafter.py +70 -0
- brain/elicit/queue.py +220 -0
- brain/elicit/schema.py +48 -0
- brain/elicit/session.py +445 -0
- brain/embedding_targets.py +54 -0
- brain/embeddings.py +424 -0
- brain/enrichment.py +808 -0
- brain/errors.py +357 -0
- brain/eval/__init__.py +129 -0
- brain/eval/answer_eval.py +281 -0
- brain/eval/baseline.py +265 -0
- brain/eval/concept_extraction.py +378 -0
- brain/eval/corpus.py +152 -0
- brain/eval/errors.py +19 -0
- brain/eval/graph_baseline.py +226 -0
- brain/eval/graph_retrieval.py +202 -0
- brain/eval/graph_runner.py +319 -0
- brain/eval/metrics.py +101 -0
- brain/eval/runner.py +223 -0
- brain/format.py +783 -0
- brain/gaps.py +390 -0
- brain/graph_rag/__init__.py +94 -0
- brain/graph_rag/_retrieval_common.py +113 -0
- brain/graph_rag/aggregates.py +303 -0
- brain/graph_rag/aliases/__init__.py +583 -0
- brain/graph_rag/backends/__init__.py +10 -0
- brain/graph_rag/backends/_age_helpers.py +473 -0
- brain/graph_rag/backends/age.py +782 -0
- brain/graph_rag/backends/base.py +272 -0
- brain/graph_rag/build.py +344 -0
- brain/graph_rag/communities.py +644 -0
- brain/graph_rag/communities_summary.py +437 -0
- brain/graph_rag/concepts.py +202 -0
- brain/graph_rag/cooccur.py +193 -0
- brain/graph_rag/cross_type.py +312 -0
- brain/graph_rag/extract.py +885 -0
- brain/graph_rag/fuse.py +371 -0
- brain/graph_rag/global_.py +412 -0
- brain/graph_rag/grouping.py +372 -0
- brain/graph_rag/person_resolver.py +167 -0
- brain/graph_rag/reconcile.py +792 -0
- brain/graph_rag/relational.py +353 -0
- brain/graph_rag/retrieve.py +526 -0
- brain/graph_rag/router.py +288 -0
- brain/graph_rag/schema.py +320 -0
- brain/graph_rag/sync.py +237 -0
- brain/graph_rag/tenancy.py +43 -0
- brain/graph_rag/themes.py +501 -0
- brain/graph_rag/weighting.py +202 -0
- brain/ingest/__init__.py +1926 -0
- brain/ingest/chunker.py +249 -0
- brain/ingest/docx.py +40 -0
- brain/ingest/gmail.py +621 -0
- brain/ingest/markdown.py +37 -0
- brain/ingest/pdf.py +61 -0
- brain/ingest/stdin.py +22 -0
- brain/ingest/sub_tokens.py +91 -0
- brain/ingest/text.py +16 -0
- brain/interactions.py +205 -0
- brain/maintenance.py +355 -0
- brain/mcp_server.py +3405 -0
- brain/migrations/001_init.sql +43 -0
- brain/migrations/002_qwen3_embedding.sql +17 -0
- brain/migrations/003_vault_model.sql +41 -0
- brain/migrations/004_relax_content_hash_uniqueness.sql +18 -0
- brain/migrations/005_derived_links.sql +67 -0
- brain/migrations/006_dedup_file_by_source_path.sql +25 -0
- brain/migrations/007_email_thread_and_draft.sql +15 -0
- brain/migrations/008_gmail_thread_unique.sql +11 -0
- brain/migrations/009_chunks_weighted_tsv.sql +28 -0
- brain/migrations/010_interactions.sql +30 -0
- brain/migrations/011_documents_summary.sql +23 -0
- brain/migrations/012_graphrag.sql +171 -0
- brain/migrations/013_graphrag_communities.sql +125 -0
- brain/migrations/014_graphrag_community_summary_hash.sql +33 -0
- brain/migrations/015_interactions_graph_targets.sql +89 -0
- brain/migrations/016_index_hygiene.sql +61 -0
- brain/migrations/017_elicit.sql +30 -0
- brain/migrations/018_review_gap_signal_kinds.sql +40 -0
- brain/migrations/019_search_queries.sql +35 -0
- brain/migrations/020_link_suggestions.sql +40 -0
- brain/migrations/021_timeline_doc_date.sql +34 -0
- brain/migrations/022_link_suggestions_undirected.sql +84 -0
- brain/migrations/023_search_queries_fts_count.sql +28 -0
- brain/quartz_overrides/__init__.py +8 -0
- brain/quartz_overrides/quartz/bootstrap-cli.mjs +65 -0
- brain/quartz_overrides/quartz/build.ts +568 -0
- brain/quartz_overrides/quartz/cli/args.js +152 -0
- brain/quartz_overrides/quartz/cli/build_partial_handler.js +544 -0
- brain/quartz_overrides/quartz/cli/handlers.js +636 -0
- brain/quartz_overrides/quartz/components/CommandPalette.tsx +172 -0
- brain/quartz_overrides/quartz/components/Explorer.tsx +198 -0
- brain/quartz_overrides/quartz/components/Footer.tsx +27 -0
- brain/quartz_overrides/quartz/components/Graph.tsx +468 -0
- brain/quartz_overrides/quartz/components/PageTitle.tsx +72 -0
- brain/quartz_overrides/quartz/components/RelatedDocs.tsx +38 -0
- brain/quartz_overrides/quartz/components/Search.tsx +161 -0
- brain/quartz_overrides/quartz/components/SummaryLede.tsx +72 -0
- brain/quartz_overrides/quartz/components/index.ts +92 -0
- brain/quartz_overrides/quartz/components/pages/TagContent.tsx +272 -0
- brain/quartz_overrides/quartz/components/scripts/commandPalette.inline.ts +665 -0
- brain/quartz_overrides/quartz/components/scripts/explorer.inline.ts +768 -0
- brain/quartz_overrides/quartz/components/scripts/graph.inline.ts +2302 -0
- brain/quartz_overrides/quartz/components/scripts/relatedDocs.inline.ts +163 -0
- brain/quartz_overrides/quartz/components/scripts/search.inline.ts +1011 -0
- brain/quartz_overrides/quartz/plugins/emitters/contentIndex.ts +546 -0
- brain/quartz_overrides/quartz/plugins/transformers/codeCopy.ts +94 -0
- brain/quartz_overrides/quartz/plugins/transformers/derivedFenceMark.ts +302 -0
- brain/quartz_overrides/quartz/plugins/transformers/emailThread.ts +148 -0
- brain/quartz_overrides/quartz/plugins/transformers/emptyDoorFilter.ts +213 -0
- brain/quartz_overrides/quartz/plugins/transformers/index.ts +114 -0
- brain/quartz_overrides/quartz/plugins/transformers/linkKindMark.ts +205 -0
- brain/quartz_overrides/quartz/plugins/transformers/linkSourceTag.ts +104 -0
- brain/quartz_overrides/quartz/plugins/transformers/relativeDate.ts +100 -0
- brain/quartz_overrides/quartz/plugins/transformers/reloadSignal.ts +131 -0
- brain/quartz_overrides/quartz/processors/parse.ts +371 -0
- brain/quartz_overrides/quartz/processors/parser_cache.ts +78 -0
- brain/quartz_overrides/quartz/static/brain-logo-dark.png +0 -0
- brain/quartz_overrides/quartz/static/brain-logo-light.png +0 -0
- brain/quartz_overrides/quartz/static/codeCopy.js +196 -0
- brain/quartz_overrides/quartz/static/emailThread.js +334 -0
- brain/quartz_overrides/quartz/static/favicon.ico +0 -0
- brain/quartz_overrides/quartz/static/icon.png +0 -0
- brain/quartz_overrides/quartz/static/linkSourceTag.js +104 -0
- brain/quartz_overrides/quartz/static/relativeDate.js +142 -0
- brain/quartz_overrides/quartz/static/reload.js +168 -0
- brain/quartz_overrides/quartz/styles/brain/_article.scss +252 -0
- brain/quartz_overrides/quartz/styles/brain/_atmosphere.scss +113 -0
- brain/quartz_overrides/quartz/styles/brain/_callouts.scss +180 -0
- brain/quartz_overrides/quartz/styles/brain/_cmdk.scss +7 -0
- brain/quartz_overrides/quartz/styles/brain/_code.scss +208 -0
- brain/quartz_overrides/quartz/styles/brain/_command_palette.scss +369 -0
- brain/quartz_overrides/quartz/styles/brain/_email_thread.scss +228 -0
- brain/quartz_overrides/quartz/styles/brain/_explorer.scss +142 -0
- brain/quartz_overrides/quartz/styles/brain/_home.scss +182 -0
- brain/quartz_overrides/quartz/styles/brain/_links.scss +322 -0
- brain/quartz_overrides/quartz/styles/brain/_marginalia.scss +117 -0
- brain/quartz_overrides/quartz/styles/brain/_motion.scss +175 -0
- brain/quartz_overrides/quartz/styles/brain/_people_hub.scss +100 -0
- brain/quartz_overrides/quartz/styles/brain/_related_docs.scss +137 -0
- brain/quartz_overrides/quartz/styles/brain/_search.scss +252 -0
- brain/quartz_overrides/quartz/styles/brain/_sidebar.scss +468 -0
- brain/quartz_overrides/quartz/styles/brain/_summary_lede.scss +56 -0
- brain/quartz_overrides/quartz/styles/brain/_surface.scss +43 -0
- brain/quartz_overrides/quartz/styles/brain/_tag_content.scss +118 -0
- brain/quartz_overrides/quartz/styles/brain/_tokens.scss +197 -0
- brain/quartz_overrides/quartz/styles/brain/_typography.scss +92 -0
- brain/quartz_overrides/quartz/styles/custom.scss +89 -0
- brain/quartz_overrides/quartz/styles/graph.scss +505 -0
- brain/quartz_overrides/quartz/util/ctx.ts +92 -0
- brain/quartz_overrides/quartz/util/fastpath_manifest.ts +608 -0
- brain/quartz_overrides/quartz/util/path.ts +358 -0
- brain/quartz_overrides/quartz/util/sourceIcons.ts +55 -0
- brain/quartz_overrides/quartz.config.ts +270 -0
- brain/quartz_overrides/quartz.layout.ts +314 -0
- brain/queries.py +1188 -0
- brain/rank_fusion.py +8 -0
- brain/resurface.py +210 -0
- brain/review/__init__.py +26 -0
- brain/review/emit.py +27 -0
- brain/review/queries.py +436 -0
- brain/review/render.py +196 -0
- brain/review/scans.py +355 -0
- brain/review/weekly.py +413 -0
- brain/search.py +704 -0
- brain/set_similarity.py +15 -0
- brain/setup.py +1205 -0
- brain/tags.py +56 -0
- brain/templates/Caddyfile.j2 +9 -0
- brain/templates/__init__.py +1 -0
- brain/templates/bin/__init__.py +1 -0
- brain/templates/bin/_brain-brief-fg.sh +25 -0
- brain/templates/bin/_brain-build-fg.sh +53 -0
- brain/templates/bin/_brain-watcher-fg.sh +65 -0
- brain/templates/bin/brain-down.sh +89 -0
- brain/templates/bin/brain-status.sh +83 -0
- brain/templates/bin/brain-up.sh +221 -0
- brain/templates/docker/age/Dockerfile +79 -0
- brain/templates/docker-compose.stock.yml.j2 +26 -0
- brain/templates/docker-compose.yml.j2 +34 -0
- brain/templates/env.example +190 -0
- brain/templates/launchd/__init__.py +1 -0
- brain/templates/launchd/com.brain.brief.plist.j2 +45 -0
- brain/templates/launchd/com.brain.build.plist.j2 +46 -0
- brain/templates/launchd/com.brain.watcher.plist.j2 +46 -0
- brain/templates/skill/SKILL.md +63 -0
- brain/templates/skill/__init__.py +1 -0
- brain/timeline.py +834 -0
- brain/todo.py +124 -0
- brain/uninstall.py +185 -0
- brain/vault/__init__.py +115 -0
- brain/vault/_atomic.py +25 -0
- brain/vault/daily_index.py +228 -0
- brain/vault/derived_links/__init__.py +50 -0
- brain/vault/derived_links/directory.py +683 -0
- brain/vault/derived_links/fence.py +408 -0
- brain/vault/derived_links/gws.py +64 -0
- brain/vault/derived_links/participants.py +143 -0
- brain/vault/derived_links/pass_runner.py +362 -0
- brain/vault/derived_links/rules.py +137 -0
- brain/vault/export.py +683 -0
- brain/vault/frontmatter.py +165 -0
- brain/vault/graph.py +620 -0
- brain/vault/graph_format.py +388 -0
- brain/vault/link_rewrite.py +235 -0
- brain/vault/links.py +260 -0
- brain/vault/note_builder.py +211 -0
- brain/vault/paths.py +55 -0
- brain/vault/quartz_overlay.py +236 -0
- brain/vault/rename.py +591 -0
- brain/vault/resolver.py +304 -0
- brain/vault/slug.py +127 -0
- brain/vault/sync.py +1513 -0
- brain/vault/sync_summaries.py +264 -0
- brain/vault/templates.py +145 -0
- brain/vault/watch.py +1052 -0
- brain/wiki/__init__.py +6 -0
- brain/wiki/_github_slugger.py +76 -0
- brain/wiki/_person_name.py +314 -0
- brain/wiki/build_homepage.py +541 -0
- brain/wiki/build_partial.py +273 -0
- brain/wiki/build_people.py +934 -0
- brain/wiki/build_related.py +758 -0
- brain/wiki/build_swap.py +585 -0
- brain/wiki/build_watcher.py +975 -0
- brain/wiki/edit_classifier.py +215 -0
- brain/wiki/errors.py +10 -0
- brain/wiki/fastpath_manifest.py +475 -0
- brain/wiki/fastpath_state.py +174 -0
- brain/wiki/install.py +296 -0
- brain/wiki/slug.py +111 -0
- secondbrain_py-0.2.1.dist-info/METADATA +195 -0
- secondbrain_py-0.2.1.dist-info/RECORD +273 -0
- secondbrain_py-0.2.1.dist-info/WHEEL +5 -0
- secondbrain_py-0.2.1.dist-info/entry_points.txt +11 -0
- secondbrain_py-0.2.1.dist-info/licenses/LICENSE +21 -0
- secondbrain_py-0.2.1.dist-info/top_level.txt +1 -0
brain/vault/export.py
ADDED
|
@@ -0,0 +1,683 @@
|
|
|
1
|
+
"""One-shot DB → vault export.
|
|
2
|
+
|
|
3
|
+
Walks ``documents`` in batches and materializes each row as a Markdown file
|
|
4
|
+
with YAML frontmatter under the vault root. Idempotent: a re-run skips files
|
|
5
|
+
whose existing content_hash on disk matches the document's content_hash in
|
|
6
|
+
the DB.
|
|
7
|
+
|
|
8
|
+
Older manual ingests have no ``sources`` row (no upstream identifier). For
|
|
9
|
+
those we synthesize a ``local-<short-uuid>`` placeholder in the filename so
|
|
10
|
+
two such docs with the same date + title can still be disambiguated.
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import hashlib
|
|
15
|
+
from collections.abc import Iterator
|
|
16
|
+
from dataclasses import dataclass, field
|
|
17
|
+
from datetime import date, datetime
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from typing import Any
|
|
20
|
+
|
|
21
|
+
import psycopg
|
|
22
|
+
import yaml
|
|
23
|
+
|
|
24
|
+
from . import init_vault
|
|
25
|
+
from ._atomic import atomic_write_text
|
|
26
|
+
from .frontmatter import dump_frontmatter, parse_frontmatter
|
|
27
|
+
from .slug import gmail_slug, slugify
|
|
28
|
+
|
|
29
|
+
_BATCH_SIZE = 100
|
|
30
|
+
_MAX_SHORT_ID = 8 # first N chars of an external/document UUID for filename use
|
|
31
|
+
_EXPORT_OWNED_FRONTMATTER_KEYS = frozenset(
|
|
32
|
+
{
|
|
33
|
+
"id",
|
|
34
|
+
"title",
|
|
35
|
+
"tags",
|
|
36
|
+
"aliases",
|
|
37
|
+
"kind",
|
|
38
|
+
"content_type",
|
|
39
|
+
"source",
|
|
40
|
+
"external_id",
|
|
41
|
+
"created",
|
|
42
|
+
"updated",
|
|
43
|
+
"vault_path",
|
|
44
|
+
# NOTE: ``summary`` is intentionally NOT in this strip set.
|
|
45
|
+
# Wave Q2-SUMMARY-WIKI's first cut stripped it always (the
|
|
46
|
+
# documented theory was that vault-tier hand-authored
|
|
47
|
+
# ``summary:`` lines round-trip through ``documents.summary``).
|
|
48
|
+
# That theory was wrong — sync's ``_build_metadata`` puts every
|
|
49
|
+
# non-reserved frontmatter key into ``documents.metadata`` for
|
|
50
|
+
# vault-tier rows, so a user's ``summary:`` lands in
|
|
51
|
+
# ``metadata["summary"]``, NOT in ``documents.summary``. Stripping
|
|
52
|
+
# it here deleted the user's content on the next export.
|
|
53
|
+
# Codex finding 2 (2026-05-11) caught the regression. The fix:
|
|
54
|
+
# leave ``summary`` in freeform passthrough for vault-tier; let
|
|
55
|
+
# :func:`_build_frontmatter` resolve precedence (Q1-D
|
|
56
|
+
# ``documents.summary`` wins; vault-tier metadata.summary serves
|
|
57
|
+
# as the fallback when ``documents.summary`` is NULL).
|
|
58
|
+
}
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@dataclass
|
|
63
|
+
class ExportSummary:
|
|
64
|
+
"""Outcome counts for ``brain vault export``."""
|
|
65
|
+
|
|
66
|
+
written: int = 0
|
|
67
|
+
skipped: int = 0
|
|
68
|
+
errors: list[str] = field(default_factory=list)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
@dataclass
|
|
72
|
+
class _DocumentForExport:
|
|
73
|
+
"""Internal projection of a documents row plus its source kind/external id."""
|
|
74
|
+
|
|
75
|
+
id: str
|
|
76
|
+
title: str
|
|
77
|
+
content: str
|
|
78
|
+
content_hash: str
|
|
79
|
+
content_type: str
|
|
80
|
+
tags: list[str]
|
|
81
|
+
metadata: dict[str, Any]
|
|
82
|
+
ingested_at: datetime | None
|
|
83
|
+
kind: str
|
|
84
|
+
vault_path: str | None
|
|
85
|
+
source_kind: str | None
|
|
86
|
+
source_external_id: str | None
|
|
87
|
+
draft: bool
|
|
88
|
+
# Wave Q2-SUMMARY-WIKI: ``documents.summary`` written by the Q1-D
|
|
89
|
+
# ``OllamaEnricher`` post-ingest hook. Plumbed onto the export
|
|
90
|
+
# projection so :func:`_build_frontmatter` can emit it as a wiki-
|
|
91
|
+
# readable field. ``None`` for any doc the enricher hasn't touched
|
|
92
|
+
# yet (short docs, Ollama unavailable, ``enrich=False``).
|
|
93
|
+
summary: str | None
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _is_directory_unmanaged(target: Path) -> bool:
|
|
97
|
+
"""True iff ``target`` is non-empty AND not a previously-initialized vault.
|
|
98
|
+
|
|
99
|
+
A "managed" vault is one that already contains the ``README.md`` written
|
|
100
|
+
by :func:`init_vault`. Anything else with files in it is unmanaged — the
|
|
101
|
+
user has to opt in with ``--force`` so we never silently scribble into
|
|
102
|
+
an unrelated folder.
|
|
103
|
+
|
|
104
|
+
Dotfiles and hidden directories (``.git/``, ``.DS_Store``, etc.) are
|
|
105
|
+
intentionally excluded from the emptiness check. A folder that contains
|
|
106
|
+
only a ``.git`` directory still counts as empty for our purposes — the
|
|
107
|
+
user almost certainly wants to use a git-tracked vault, and forcing
|
|
108
|
+
them through ``--force`` for an empty initial repo would be obnoxious.
|
|
109
|
+
"""
|
|
110
|
+
if not target.exists():
|
|
111
|
+
return False
|
|
112
|
+
if not target.is_dir():
|
|
113
|
+
return True
|
|
114
|
+
contents = [p for p in target.iterdir() if not p.name.startswith(".")]
|
|
115
|
+
if not contents:
|
|
116
|
+
return False
|
|
117
|
+
# Treat the presence of vault README.md (signature of init_vault) as the
|
|
118
|
+
# "yes, this is our folder" marker.
|
|
119
|
+
return not (target / "README.md").is_file()
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
_DOCUMENT_FOR_EXPORT_COLUMNS = (
|
|
123
|
+
"d.id::text, d.title, d.content, d.content_hash, "
|
|
124
|
+
"d.content_type, d.tags, d.metadata, d.ingested_at, "
|
|
125
|
+
"d.kind, d.vault_path, s.kind, s.external_id, d.draft, "
|
|
126
|
+
# Wave Q2-SUMMARY-WIKI: pull the Q1-D auto-summary onto the export
|
|
127
|
+
# projection so :func:`_build_frontmatter` can emit it as a
|
|
128
|
+
# ``summary:`` field in the per-doc mirror.
|
|
129
|
+
"d.summary"
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _row_to_document_for_export(row: tuple[Any, ...]) -> _DocumentForExport:
|
|
134
|
+
"""Project a SELECT row into the dataclass.
|
|
135
|
+
|
|
136
|
+
Centralizes the column-index mapping so :func:`_iter_documents` and
|
|
137
|
+
:func:`_fetch_document_for_export` share one source of truth and a
|
|
138
|
+
schema change touches a single helper.
|
|
139
|
+
"""
|
|
140
|
+
return _DocumentForExport(
|
|
141
|
+
id=str(row[0]),
|
|
142
|
+
title=str(row[1]),
|
|
143
|
+
content=str(row[2]),
|
|
144
|
+
content_hash=str(row[3]),
|
|
145
|
+
content_type=str(row[4]),
|
|
146
|
+
tags=list(row[5] or []),
|
|
147
|
+
metadata=dict(row[6] or {}),
|
|
148
|
+
ingested_at=row[7],
|
|
149
|
+
kind=str(row[8]),
|
|
150
|
+
vault_path=row[9],
|
|
151
|
+
source_kind=row[10],
|
|
152
|
+
source_external_id=row[11],
|
|
153
|
+
draft=bool(row[12]),
|
|
154
|
+
# ``documents.summary`` is NULL until the Q1-D enrich hook
|
|
155
|
+
# populates it (or a future ``brain enrich --backfill`` run).
|
|
156
|
+
# Coerce to ``str`` only when set so ``_build_frontmatter`` can
|
|
157
|
+
# gate the emit on a truthy check.
|
|
158
|
+
summary=str(row[13]) if row[13] is not None else None,
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def _iter_documents(conn: psycopg.Connection[Any]) -> Iterator[_DocumentForExport]:
|
|
163
|
+
"""Yield every document, joined with its source row, in batches.
|
|
164
|
+
|
|
165
|
+
Keyset pagination on ``documents.id`` keeps memory bounded even for very
|
|
166
|
+
large corpora; the batch is materialized in Python (the connection is
|
|
167
|
+
free for follow-up writes / lookups while the iterator is in flight).
|
|
168
|
+
"""
|
|
169
|
+
last_id: str | None = None
|
|
170
|
+
while True:
|
|
171
|
+
if last_id is None:
|
|
172
|
+
rows = conn.execute(
|
|
173
|
+
f"""
|
|
174
|
+
SELECT {_DOCUMENT_FOR_EXPORT_COLUMNS}
|
|
175
|
+
FROM documents d
|
|
176
|
+
LEFT JOIN sources s ON s.id = d.source_id
|
|
177
|
+
ORDER BY d.id
|
|
178
|
+
LIMIT %s
|
|
179
|
+
""",
|
|
180
|
+
(_BATCH_SIZE,),
|
|
181
|
+
).fetchall()
|
|
182
|
+
else:
|
|
183
|
+
rows = conn.execute(
|
|
184
|
+
f"""
|
|
185
|
+
SELECT {_DOCUMENT_FOR_EXPORT_COLUMNS}
|
|
186
|
+
FROM documents d
|
|
187
|
+
LEFT JOIN sources s ON s.id = d.source_id
|
|
188
|
+
WHERE d.id > %s::uuid
|
|
189
|
+
ORDER BY d.id
|
|
190
|
+
LIMIT %s
|
|
191
|
+
""",
|
|
192
|
+
(last_id, _BATCH_SIZE),
|
|
193
|
+
).fetchall()
|
|
194
|
+
if not rows:
|
|
195
|
+
return
|
|
196
|
+
last_id = str(rows[-1][0])
|
|
197
|
+
for r in rows:
|
|
198
|
+
yield _row_to_document_for_export(r)
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def _fetch_document_for_export(
|
|
202
|
+
conn: psycopg.Connection[Any], document_id: str
|
|
203
|
+
) -> _DocumentForExport | None:
|
|
204
|
+
"""Fetch a single ``documents`` row in the export projection.
|
|
205
|
+
|
|
206
|
+
Returns ``None`` when no row matches ``document_id``. Mirrors the column
|
|
207
|
+
list used by :func:`_iter_documents` so the resulting dataclass is
|
|
208
|
+
populated identically for both code paths.
|
|
209
|
+
"""
|
|
210
|
+
row = conn.execute(
|
|
211
|
+
f"""
|
|
212
|
+
SELECT {_DOCUMENT_FOR_EXPORT_COLUMNS}
|
|
213
|
+
FROM documents d
|
|
214
|
+
LEFT JOIN sources s ON s.id = d.source_id
|
|
215
|
+
WHERE d.id = %s::uuid
|
|
216
|
+
""",
|
|
217
|
+
(document_id,),
|
|
218
|
+
).fetchone()
|
|
219
|
+
if row is None:
|
|
220
|
+
return None
|
|
221
|
+
return _row_to_document_for_export(row)
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def _short_id(value: str) -> str:
|
|
225
|
+
"""Return the first 8 hex chars of a UUID-like value (no surrounding dashes).
|
|
226
|
+
|
|
227
|
+
Used both for the external-id slot in filenames and for the collision
|
|
228
|
+
suffix. Stripping dashes first makes ``-aabbccdd`` collisions deterministic
|
|
229
|
+
regardless of the full UUID's hyphenation style. For non-UUID
|
|
230
|
+
``external_id`` values (e.g. Slack's ``1234567.890`` timestamps) the
|
|
231
|
+
strip-then-truncate behavior preserves the leading bytes verbatim once
|
|
232
|
+
any non-hex separator characters have been dropped.
|
|
233
|
+
"""
|
|
234
|
+
return value.replace("-", "")[:_MAX_SHORT_ID]
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def _date_prefix(doc: _DocumentForExport) -> str:
|
|
238
|
+
"""Pick the YYYY-MM-DD prefix for an ingested file's filename.
|
|
239
|
+
|
|
240
|
+
``metadata.date`` wins when present (a string ISO date or datetime, or a
|
|
241
|
+
date/datetime object). Otherwise we fall back to ``ingested_at::date``,
|
|
242
|
+
which is non-NULL by schema (``documents.ingested_at NOT NULL DEFAULT
|
|
243
|
+
NOW()``) — the assert is defensive so a future schema regression
|
|
244
|
+
surfaces here rather than producing a malformed filename.
|
|
245
|
+
"""
|
|
246
|
+
raw = doc.metadata.get("date")
|
|
247
|
+
if isinstance(raw, str) and raw:
|
|
248
|
+
# Accept "2026-04-15" or "2026-04-15T..." — the date is the first 10 chars.
|
|
249
|
+
return raw[:10]
|
|
250
|
+
if isinstance(raw, (datetime, date)):
|
|
251
|
+
return raw.strftime("%Y-%m-%d") if isinstance(raw, date) else raw.date().isoformat()
|
|
252
|
+
assert doc.ingested_at is not None, "documents.ingested_at is NOT NULL"
|
|
253
|
+
return doc.ingested_at.date().isoformat()
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def _parse_iso_datetime(raw: Any) -> datetime | None:
|
|
257
|
+
"""Parse an ISO-8601 string (with optional trailing ``Z``) into a datetime.
|
|
258
|
+
|
|
259
|
+
Returns ``None`` for any non-string / unparseable input. Used by the
|
|
260
|
+
Gmail-slug branch to read ``metadata.sent_at`` (set by the gmail
|
|
261
|
+
extractor). Naive datetimes survive as-is — :func:`gmail_slug` treats
|
|
262
|
+
them as UTC.
|
|
263
|
+
"""
|
|
264
|
+
if not isinstance(raw, str) or not raw.strip():
|
|
265
|
+
return None
|
|
266
|
+
try:
|
|
267
|
+
return datetime.fromisoformat(raw.replace("Z", "+00:00"))
|
|
268
|
+
except ValueError:
|
|
269
|
+
return None
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def _gmail_relative_path(doc: _DocumentForExport) -> str | None:
|
|
273
|
+
"""Compute the ``_ingested/gmail/<gmail-slug>.md`` path for a Gmail doc.
|
|
274
|
+
|
|
275
|
+
Returns ``None`` when the metadata is too sparse to build a stable slug
|
|
276
|
+
(no ``thread_id`` AND no usable date) — the caller falls back to the
|
|
277
|
+
generic ingested-path rules so legacy rows still export. Idempotent: a
|
|
278
|
+
given ``(thread_id, sent_at, subject)`` triple always yields the same
|
|
279
|
+
path.
|
|
280
|
+
"""
|
|
281
|
+
thread_id = doc.metadata.get("thread_id")
|
|
282
|
+
if not isinstance(thread_id, str) or not thread_id:
|
|
283
|
+
return None
|
|
284
|
+
sent_at = _parse_iso_datetime(doc.metadata.get("sent_at"))
|
|
285
|
+
if sent_at is None and doc.ingested_at is None:
|
|
286
|
+
return None
|
|
287
|
+
slug = gmail_slug(
|
|
288
|
+
thread_id,
|
|
289
|
+
sent_at,
|
|
290
|
+
doc.title,
|
|
291
|
+
fallback_date=doc.ingested_at,
|
|
292
|
+
)
|
|
293
|
+
return f"_ingested/gmail/{slug}.md"
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
def _ingested_relative_path(
|
|
297
|
+
doc: _DocumentForExport, used_paths: set[str]
|
|
298
|
+
) -> str:
|
|
299
|
+
"""Compute the ``_ingested/<source>/...`` path for an ingested-tier doc.
|
|
300
|
+
|
|
301
|
+
The shape depends on the source:
|
|
302
|
+
|
|
303
|
+
- ``manual``: ``_ingested/manual/<slug>.md`` (no date prefix; older
|
|
304
|
+
manual ingests had no upstream date at all).
|
|
305
|
+
- ``gmail``: ``_ingested/gmail/<gmail-slug>.md`` where ``<gmail-slug>``
|
|
306
|
+
is :func:`brain.vault.slug.gmail_slug` over
|
|
307
|
+
``(thread_id, sent_at, subject)``. Stable across re-ingests of the
|
|
308
|
+
same thread; falls back to the generic shape below for legacy rows
|
|
309
|
+
missing ``thread_id``.
|
|
310
|
+
- everything else: ``_ingested/<source>/<YYYY-MM-DD>-<external-id>-<slug>.md``
|
|
311
|
+
using the first 8 chars of ``sources.external_id`` (or
|
|
312
|
+
``local-<short-doc-id>`` when no source row exists, e.g. legacy
|
|
313
|
+
manual ingests under any kind).
|
|
314
|
+
|
|
315
|
+
On collision (two docs hashing to the same path) we append the short
|
|
316
|
+
document UUID — stable across re-runs because it's keyed off the immutable
|
|
317
|
+
``documents.id``.
|
|
318
|
+
"""
|
|
319
|
+
source = doc.source_kind or "manual"
|
|
320
|
+
slug = slugify(doc.title)
|
|
321
|
+
|
|
322
|
+
if source == "manual":
|
|
323
|
+
candidate = f"_ingested/manual/{slug}.md"
|
|
324
|
+
elif source == "gmail":
|
|
325
|
+
gmail_path = _gmail_relative_path(doc)
|
|
326
|
+
if gmail_path is not None:
|
|
327
|
+
candidate = gmail_path
|
|
328
|
+
else:
|
|
329
|
+
date_prefix = _date_prefix(doc)
|
|
330
|
+
external = (
|
|
331
|
+
_short_id(doc.source_external_id)
|
|
332
|
+
if doc.source_external_id
|
|
333
|
+
else f"local-{_short_id(doc.id)}"
|
|
334
|
+
)
|
|
335
|
+
candidate = f"_ingested/gmail/{date_prefix}-{external}-{slug}.md"
|
|
336
|
+
else:
|
|
337
|
+
date_prefix = _date_prefix(doc)
|
|
338
|
+
external = (
|
|
339
|
+
_short_id(doc.source_external_id)
|
|
340
|
+
if doc.source_external_id
|
|
341
|
+
else f"local-{_short_id(doc.id)}"
|
|
342
|
+
)
|
|
343
|
+
candidate = f"_ingested/{source}/{date_prefix}-{external}-{slug}.md"
|
|
344
|
+
|
|
345
|
+
if candidate in used_paths:
|
|
346
|
+
# Deterministic collision suffix: short hash of the doc id, never a
|
|
347
|
+
# counter (counters would shuffle every re-run).
|
|
348
|
+
suffix = _short_id(doc.id)
|
|
349
|
+
stem, ext = candidate.rsplit(".", 1)
|
|
350
|
+
candidate = f"{stem}-{suffix}.{ext}"
|
|
351
|
+
return candidate
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
def _resolve_relative_path(
|
|
355
|
+
doc: _DocumentForExport, used_paths: set[str]
|
|
356
|
+
) -> str:
|
|
357
|
+
"""Pick the on-disk path for ``doc``.
|
|
358
|
+
|
|
359
|
+
Rows with an explicit ``vault_path`` round-trip identity (vault-tier files
|
|
360
|
+
stay where the user authored them; ingested mirrors stay at their recorded
|
|
361
|
+
canonical path). Rows without one land in ``_ingested/`` per
|
|
362
|
+
:func:`_ingested_relative_path`.
|
|
363
|
+
"""
|
|
364
|
+
if doc.vault_path:
|
|
365
|
+
return doc.vault_path
|
|
366
|
+
return _ingested_relative_path(doc, used_paths)
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
def _build_frontmatter(doc: _DocumentForExport) -> dict[str, Any]:
|
|
370
|
+
"""Return the ordered frontmatter mapping for ``doc``.
|
|
371
|
+
|
|
372
|
+
Field order is intentional and stable: id, title, created, updated, tags,
|
|
373
|
+
aliases (when non-empty), kind, content_type, ``summary`` (when the Q1-D
|
|
374
|
+
enricher has populated ``documents.summary``), then ingested-tier extras
|
|
375
|
+
(source / external_id) when present, and finally ``draft: true`` when the
|
|
376
|
+
document is quarantined. Anything not applicable (e.g. ingested extras
|
|
377
|
+
for a vault-tier doc, an empty alias list, or a NULL summary) is omitted
|
|
378
|
+
entirely rather than written as ``null`` / ``[]`` — keeps the vault file
|
|
379
|
+
readable and round-trips cleanly through sync.
|
|
380
|
+
|
|
381
|
+
The ``draft`` line is emitted ONLY when ``documents.draft`` is true. The
|
|
382
|
+
default-false case is the vast majority of files; writing ``draft: false``
|
|
383
|
+
on every export would be visual noise. The Quartz contentIndex emitter
|
|
384
|
+
reads this key (or the absence of it) to filter quarantined docs out of
|
|
385
|
+
the wiki's explorer / graph / search index without touching the DB row.
|
|
386
|
+
|
|
387
|
+
The ``summary`` line (wave Q2-SUMMARY-WIKI) is emitted whenever
|
|
388
|
+
``documents.summary`` is non-NULL — that is the same surface
|
|
389
|
+
``brain show`` / MCP ``brain_show`` expose. The Quartz
|
|
390
|
+
``SummaryLede`` component renders it as an inline TL;DR above the
|
|
391
|
+
article body. Summaries are NOT indexed (the Q1-D spec keeps them
|
|
392
|
+
out of tsv / embeddings / SearchExplanation); this writer is the
|
|
393
|
+
read-surface pipeline only.
|
|
394
|
+
|
|
395
|
+
Vault-tier rows also preserve freeform metadata keys because sync stores
|
|
396
|
+
user-authored frontmatter there. Export-owned keys stay canonical here,
|
|
397
|
+
and ingested-tier metadata remains DB-only so source metadata does not
|
|
398
|
+
leak into mirror frontmatter.
|
|
399
|
+
|
|
400
|
+
Aliases come from ``documents.metadata['aliases']`` (the canonical
|
|
401
|
+
storage location per the spec) — only string elements are emitted; any
|
|
402
|
+
non-string value in the array is dropped silently to keep the
|
|
403
|
+
frontmatter's ``aliases:`` always be a flat list of strings.
|
|
404
|
+
"""
|
|
405
|
+
fields: dict[str, Any] = {
|
|
406
|
+
"id": doc.id,
|
|
407
|
+
"title": doc.title,
|
|
408
|
+
}
|
|
409
|
+
if doc.ingested_at is not None:
|
|
410
|
+
iso = doc.ingested_at.isoformat()
|
|
411
|
+
fields["created"] = iso
|
|
412
|
+
fields["updated"] = iso
|
|
413
|
+
fields["tags"] = list(doc.tags)
|
|
414
|
+
aliases = _aliases_from_metadata(doc.metadata)
|
|
415
|
+
if aliases:
|
|
416
|
+
fields["aliases"] = aliases
|
|
417
|
+
fields["kind"] = doc.kind
|
|
418
|
+
fields["content_type"] = doc.content_type
|
|
419
|
+
# Wave Q2-SUMMARY-WIKI: emit ``summary`` with tier-aware precedence.
|
|
420
|
+
# 1. ``documents.summary`` (Q1-D ``OllamaEnricher``) — authoritative
|
|
421
|
+
# whenever non-NULL. The wiki's ``SummaryLede`` reads this key.
|
|
422
|
+
# 2. For vault-tier docs only, fall back to a hand-authored
|
|
423
|
+
# ``summary:`` line that sync stored in ``documents.metadata``.
|
|
424
|
+
# Ingested-tier docs never fall back — their metadata is
|
|
425
|
+
# DB-authoritative, not user-authored.
|
|
426
|
+
# The early write here pins the field's position right after
|
|
427
|
+
# ``content_type`` so YAML field order stays stable regardless of
|
|
428
|
+
# which source the value came from. The later vault-tier freeform
|
|
429
|
+
# merge may overwrite this with the metadata value; we re-assert
|
|
430
|
+
# ``documents.summary`` after the merge to keep it authoritative.
|
|
431
|
+
effective_summary: str | None = doc.summary
|
|
432
|
+
if effective_summary is None and doc.kind == "vault":
|
|
433
|
+
metadata_summary = doc.metadata.get("summary")
|
|
434
|
+
if isinstance(metadata_summary, str) and metadata_summary:
|
|
435
|
+
effective_summary = metadata_summary
|
|
436
|
+
if effective_summary:
|
|
437
|
+
fields["summary"] = effective_summary
|
|
438
|
+
if doc.kind == "ingested":
|
|
439
|
+
if doc.source_kind:
|
|
440
|
+
fields["source"] = doc.source_kind
|
|
441
|
+
if doc.source_external_id:
|
|
442
|
+
fields["external_id"] = doc.source_external_id
|
|
443
|
+
if doc.kind == "vault":
|
|
444
|
+
fields.update(_freeform_vault_metadata(doc.metadata))
|
|
445
|
+
# Re-assert Q1-D precedence: the freeform merge above may have
|
|
446
|
+
# overwritten ``fields["summary"]`` with ``metadata["summary"]``
|
|
447
|
+
# (Codex finding 2 fix removed ``summary`` from the strip set so
|
|
448
|
+
# vault-tier hand-authored values survive). When
|
|
449
|
+
# ``documents.summary`` is also set, it wins — re-pin it at the
|
|
450
|
+
# field's existing dict position so YAML order is unchanged.
|
|
451
|
+
if doc.summary:
|
|
452
|
+
fields["summary"] = doc.summary
|
|
453
|
+
if doc.draft:
|
|
454
|
+
fields["draft"] = True
|
|
455
|
+
return fields
|
|
456
|
+
|
|
457
|
+
|
|
458
|
+
def _freeform_vault_metadata(metadata: dict[str, Any]) -> dict[str, Any]:
|
|
459
|
+
"""Return vault-tier metadata keys that should round-trip to YAML."""
|
|
460
|
+
out: dict[str, Any] = {}
|
|
461
|
+
for key, value in metadata.items():
|
|
462
|
+
if key in _EXPORT_OWNED_FRONTMATTER_KEYS:
|
|
463
|
+
continue
|
|
464
|
+
out[key] = value
|
|
465
|
+
return out
|
|
466
|
+
|
|
467
|
+
|
|
468
|
+
def _aliases_from_metadata(metadata: dict[str, Any]) -> list[str]:
|
|
469
|
+
"""Extract a clean list of alias strings from ``documents.metadata``.
|
|
470
|
+
|
|
471
|
+
Only emits the field when the metadata payload actually carries one or
|
|
472
|
+
more string aliases. Anything else (missing key, non-list, list of
|
|
473
|
+
non-strings) returns an empty list — the caller then omits the
|
|
474
|
+
``aliases:`` line altogether. Defensive coercion here keeps a corrupt
|
|
475
|
+
metadata blob from poisoning the entire export pass.
|
|
476
|
+
"""
|
|
477
|
+
raw = metadata.get("aliases")
|
|
478
|
+
if not isinstance(raw, list):
|
|
479
|
+
return []
|
|
480
|
+
return [a for a in raw if isinstance(a, str) and a]
|
|
481
|
+
|
|
482
|
+
|
|
483
|
+
def _content_hash(text: str) -> str:
|
|
484
|
+
"""SHA-256 of ``text`` — matches the ingest pipeline's hashing scheme.
|
|
485
|
+
|
|
486
|
+
Idempotent re-export compares this against ``documents.content_hash`` on a
|
|
487
|
+
body re-extracted from the existing vault file (frontmatter stripped) so
|
|
488
|
+
the user can edit frontmatter (or run a re-export with a new field
|
|
489
|
+
ordering) without forcing a re-write.
|
|
490
|
+
"""
|
|
491
|
+
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
|
492
|
+
|
|
493
|
+
|
|
494
|
+
def _existing_body_hash(target: Path) -> str | None:
|
|
495
|
+
"""Read ``target`` as a vault file and return SHA-256 of its body.
|
|
496
|
+
|
|
497
|
+
Returns ``None`` if the file doesn't exist or can't be parsed (we'll
|
|
498
|
+
treat unparseable files as "needs rewrite" rather than skipping them
|
|
499
|
+
silently). Frontmatter is stripped before hashing so a frontmatter-only
|
|
500
|
+
edit doesn't trigger a needless body rewrite.
|
|
501
|
+
"""
|
|
502
|
+
if not target.is_file():
|
|
503
|
+
return None
|
|
504
|
+
try:
|
|
505
|
+
text = target.read_text(encoding="utf-8")
|
|
506
|
+
except OSError:
|
|
507
|
+
return None
|
|
508
|
+
try:
|
|
509
|
+
_, body = parse_frontmatter(text)
|
|
510
|
+
except (ValueError, yaml.YAMLError):
|
|
511
|
+
# ValueError covers our own "frontmatter must be a mapping" guard;
|
|
512
|
+
# yaml.YAMLError covers malformed YAML inside the fences. Anything
|
|
513
|
+
# broader would mask real bugs (per CLAUDE.md: always catch specific
|
|
514
|
+
# exceptions).
|
|
515
|
+
return None
|
|
516
|
+
return _content_hash(body)
|
|
517
|
+
|
|
518
|
+
|
|
519
|
+
def _write_doc_file(
|
|
520
|
+
doc: _DocumentForExport,
|
|
521
|
+
*,
|
|
522
|
+
vault_path: Path,
|
|
523
|
+
relative: str,
|
|
524
|
+
force: bool = False,
|
|
525
|
+
) -> tuple[Path, bool]:
|
|
526
|
+
"""Materialize ``doc`` at ``vault_path / relative`` and return the result.
|
|
527
|
+
|
|
528
|
+
The returned tuple is ``(target_path, written)`` — ``written`` is False
|
|
529
|
+
when the existing file's body already matches ``doc.content_hash``
|
|
530
|
+
(idempotent skip), True when the file was rewritten or freshly created.
|
|
531
|
+
|
|
532
|
+
``force=True`` bypasses the body-hash skip and rewrites the file even
|
|
533
|
+
when the body is unchanged. Callers must use ``force=True`` when they've
|
|
534
|
+
already determined a frontmatter-only change happened — otherwise the
|
|
535
|
+
body-hash check (which only fingerprints the body) silently drops the
|
|
536
|
+
rewrite and leaves stale frontmatter on disk. The default
|
|
537
|
+
``force=False`` keeps the corpus-dump path (:func:`export_vault`) cheap
|
|
538
|
+
on re-runs.
|
|
539
|
+
|
|
540
|
+
Raises :class:`OSError` if the write fails. Callers decide whether to
|
|
541
|
+
aggregate the failure into a summary or surface it directly.
|
|
542
|
+
"""
|
|
543
|
+
target = vault_path / relative
|
|
544
|
+
if not force:
|
|
545
|
+
existing_hash = _existing_body_hash(target)
|
|
546
|
+
if existing_hash == doc.content_hash:
|
|
547
|
+
return target, False
|
|
548
|
+
|
|
549
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
550
|
+
fields = _build_frontmatter(doc)
|
|
551
|
+
text = dump_frontmatter(fields, doc.content)
|
|
552
|
+
# Atomic rewrite — sibling tempfile + ``os.replace`` so a crash between
|
|
553
|
+
# the write and the rename never leaves a truncated mirror file on disk.
|
|
554
|
+
# All writeback callers (``brain edit`` / ``brain tag`` / ``brain mark-draft`` /
|
|
555
|
+
# ``brain vault sync-summaries`` / full ``brain vault export``) funnel through
|
|
556
|
+
# here, so this single line gives the whole mirror-write surface crash
|
|
557
|
+
# safety. Matches the convention used by every other body-rewriter under
|
|
558
|
+
# ``vault/`` (see :mod:`brain.vault._atomic`).
|
|
559
|
+
atomic_write_text(target, text)
|
|
560
|
+
return target, True
|
|
561
|
+
|
|
562
|
+
|
|
563
|
+
def export_vault(
|
|
564
|
+
conn: psycopg.Connection[Any],
|
|
565
|
+
*,
|
|
566
|
+
vault_path: Path,
|
|
567
|
+
force: bool = False,
|
|
568
|
+
) -> ExportSummary:
|
|
569
|
+
"""Dump every ``documents`` row to ``vault_path`` as a Markdown file.
|
|
570
|
+
|
|
571
|
+
Steps:
|
|
572
|
+
|
|
573
|
+
1. Validate the target — refuse to write into a non-empty unmanaged
|
|
574
|
+
directory unless ``force`` is true (keeps us from scribbling into an
|
|
575
|
+
arbitrary folder the user pointed us at by mistake).
|
|
576
|
+
2. Run :func:`init_vault` semantics so ``_templates/``, ``_ingested/``,
|
|
577
|
+
and the README scaffold are guaranteed to exist.
|
|
578
|
+
3. Iterate documents in keyset-paginated batches; for each one, compute
|
|
579
|
+
the destination path, build the frontmatter, and skip the write if
|
|
580
|
+
the existing file's body already matches the document's content_hash.
|
|
581
|
+
|
|
582
|
+
Collisions in derived paths are resolved by appending a short
|
|
583
|
+
document-id suffix — deterministic across re-runs because it's keyed off
|
|
584
|
+
the immutable UUID, not a counter.
|
|
585
|
+
"""
|
|
586
|
+
if _is_directory_unmanaged(vault_path) and not force:
|
|
587
|
+
raise ValueError(
|
|
588
|
+
f"target is not empty and was not created by this tool: "
|
|
589
|
+
f"{vault_path} — pass --force to write into it"
|
|
590
|
+
)
|
|
591
|
+
|
|
592
|
+
init_vault(vault_path)
|
|
593
|
+
|
|
594
|
+
summary = ExportSummary()
|
|
595
|
+
used_paths: set[str] = set()
|
|
596
|
+
|
|
597
|
+
for doc in _iter_documents(conn):
|
|
598
|
+
relative = _resolve_relative_path(doc, used_paths)
|
|
599
|
+
used_paths.add(relative)
|
|
600
|
+
try:
|
|
601
|
+
_, written = _write_doc_file(
|
|
602
|
+
doc, vault_path=vault_path, relative=relative
|
|
603
|
+
)
|
|
604
|
+
except OSError as e:
|
|
605
|
+
summary.errors.append(f"{relative}: {e}")
|
|
606
|
+
continue
|
|
607
|
+
if written:
|
|
608
|
+
summary.written += 1
|
|
609
|
+
else:
|
|
610
|
+
summary.skipped += 1
|
|
611
|
+
|
|
612
|
+
return summary
|
|
613
|
+
|
|
614
|
+
|
|
615
|
+
def regenerate_vault_file(
|
|
616
|
+
conn: psycopg.Connection[Any],
|
|
617
|
+
document_id: str,
|
|
618
|
+
*,
|
|
619
|
+
vault_path: Path,
|
|
620
|
+
force: bool = False,
|
|
621
|
+
) -> Path:
|
|
622
|
+
"""Re-create a single doc's vault mirror file from its DB row.
|
|
623
|
+
|
|
624
|
+
Returns the absolute path of the file written (or, on the idempotent
|
|
625
|
+
skip path, the path that already matched). The body-hash check from
|
|
626
|
+
:func:`export_vault` is preserved by default: if the on-disk body already
|
|
627
|
+
matches ``documents.content_hash``, the file is left untouched.
|
|
628
|
+
|
|
629
|
+
Pass ``force=True`` to bypass the body-hash skip and unconditionally
|
|
630
|
+
rewrite the file. This is the right choice when the caller has already
|
|
631
|
+
determined a change happened (e.g., a frontmatter-only edit such as a
|
|
632
|
+
title, tags, metadata, or content_type update) — the body-hash check
|
|
633
|
+
only fingerprints the body, so it would silently drop the rewrite and
|
|
634
|
+
leave stale frontmatter on disk. The ingest pipeline always sets
|
|
635
|
+
``force=True`` because it gates the call on a confirmed change
|
|
636
|
+
(created row or frontmatter-bearing field changed). The full-corpus
|
|
637
|
+
:func:`export_vault` keeps ``force=False`` so re-runs stay cheap.
|
|
638
|
+
|
|
639
|
+
Path resolution prefers the doc's existing ``vault_path`` when set —
|
|
640
|
+
a previously-synced ingested doc must regenerate at the same place
|
|
641
|
+
rather than at a freshly computed ``_ingested/<source>/...`` location
|
|
642
|
+
(which would orphan the original mirror). Docs with no ``vault_path``
|
|
643
|
+
fall back to the corpus-dump rules (``_resolve_relative_path``) with
|
|
644
|
+
a fresh, single-doc collision set.
|
|
645
|
+
|
|
646
|
+
Side effect: also UPDATEs ``documents.vault_path`` to the chosen
|
|
647
|
+
relative path so subsequent ``regenerate_vault_file``, ``brain rm``,
|
|
648
|
+
and ``brain tag`` calls all use the same on-disk location. The
|
|
649
|
+
UPDATE is its own statement under ``conn.autocommit=True`` (which
|
|
650
|
+
all current callers use) — a future caller wrapping this function
|
|
651
|
+
in a read-only transaction would have to opt that statement in.
|
|
652
|
+
The ``IS DISTINCT FROM`` predicate keeps it idempotent (NULL-safe):
|
|
653
|
+
re-running on a row that already has the right ``vault_path`` is a
|
|
654
|
+
no-op at the row level.
|
|
655
|
+
|
|
656
|
+
Raises:
|
|
657
|
+
ValueError: ``document_id`` does not match any row.
|
|
658
|
+
ValueError: the row is ``kind='vault'``. Vault-tier authored notes
|
|
659
|
+
are file-source-of-truth — regenerating from the DB risks
|
|
660
|
+
losing edits that haven't been re-synced. Restore from backup
|
|
661
|
+
or git instead.
|
|
662
|
+
OSError: the write itself fails (permissions, disk full, etc.).
|
|
663
|
+
"""
|
|
664
|
+
doc = _fetch_document_for_export(conn, document_id)
|
|
665
|
+
if doc is None:
|
|
666
|
+
raise ValueError(f"no document with id {document_id}")
|
|
667
|
+
if doc.kind == "vault":
|
|
668
|
+
raise ValueError(
|
|
669
|
+
"cannot regenerate vault-tier authored note from DB; "
|
|
670
|
+
"restore from backup or git instead"
|
|
671
|
+
)
|
|
672
|
+
|
|
673
|
+
relative = doc.vault_path or _resolve_relative_path(doc, used_paths=set())
|
|
674
|
+
|
|
675
|
+
target, _ = _write_doc_file(
|
|
676
|
+
doc, vault_path=vault_path, relative=relative, force=force
|
|
677
|
+
)
|
|
678
|
+
conn.execute(
|
|
679
|
+
"UPDATE documents SET vault_path = %s "
|
|
680
|
+
"WHERE id = %s AND vault_path IS DISTINCT FROM %s",
|
|
681
|
+
(relative, document_id, relative),
|
|
682
|
+
)
|
|
683
|
+
return target.resolve()
|