secondbrain-py 0.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- brain/__init__.py +0 -0
- brain/__main__.py +18 -0
- brain/_capture_command.py +445 -0
- brain/_compose.py +52 -0
- brain/activity.py +206 -0
- brain/ask.py +631 -0
- brain/audio.py +591 -0
- brain/backfill/__init__.py +12 -0
- brain/backfill/search_extras.py +141 -0
- brain/backfill/source_rows.py +101 -0
- brain/bin/__init__.py +1 -0
- brain/bin/_launcher.py +107 -0
- brain/bin/down.py +8 -0
- brain/bin/launchd.py +268 -0
- brain/bin/monitor.py +570 -0
- brain/bin/rebuild.py +8 -0
- brain/bin/status.py +8 -0
- brain/bin/up.py +8 -0
- brain/brief.py +272 -0
- brain/capture.py +49 -0
- brain/chat.py +293 -0
- brain/cli.py +9760 -0
- brain/cli_claude.py +81 -0
- brain/cli_connect.py +285 -0
- brain/cli_demo.py +266 -0
- brain/config.py +1949 -0
- brain/connect.py +925 -0
- brain/db.py +540 -0
- brain/demo/__init__.py +452 -0
- brain/demo/corpus/manifest.json +403 -0
- brain/demo/embedder.py +74 -0
- brain/durations.py +84 -0
- brain/edit_session.py +156 -0
- brain/editor.py +67 -0
- brain/elicit/__init__.py +16 -0
- brain/elicit/detectors.py +250 -0
- brain/elicit/drafter.py +70 -0
- brain/elicit/queue.py +220 -0
- brain/elicit/schema.py +48 -0
- brain/elicit/session.py +445 -0
- brain/embedding_targets.py +54 -0
- brain/embeddings.py +424 -0
- brain/enrichment.py +808 -0
- brain/errors.py +357 -0
- brain/eval/__init__.py +129 -0
- brain/eval/answer_eval.py +281 -0
- brain/eval/baseline.py +265 -0
- brain/eval/concept_extraction.py +378 -0
- brain/eval/corpus.py +152 -0
- brain/eval/errors.py +19 -0
- brain/eval/graph_baseline.py +226 -0
- brain/eval/graph_retrieval.py +202 -0
- brain/eval/graph_runner.py +319 -0
- brain/eval/metrics.py +101 -0
- brain/eval/runner.py +223 -0
- brain/format.py +783 -0
- brain/gaps.py +390 -0
- brain/graph_rag/__init__.py +94 -0
- brain/graph_rag/_retrieval_common.py +113 -0
- brain/graph_rag/aggregates.py +303 -0
- brain/graph_rag/aliases/__init__.py +583 -0
- brain/graph_rag/backends/__init__.py +10 -0
- brain/graph_rag/backends/_age_helpers.py +473 -0
- brain/graph_rag/backends/age.py +782 -0
- brain/graph_rag/backends/base.py +272 -0
- brain/graph_rag/build.py +344 -0
- brain/graph_rag/communities.py +644 -0
- brain/graph_rag/communities_summary.py +437 -0
- brain/graph_rag/concepts.py +202 -0
- brain/graph_rag/cooccur.py +193 -0
- brain/graph_rag/cross_type.py +312 -0
- brain/graph_rag/extract.py +885 -0
- brain/graph_rag/fuse.py +371 -0
- brain/graph_rag/global_.py +412 -0
- brain/graph_rag/grouping.py +372 -0
- brain/graph_rag/person_resolver.py +167 -0
- brain/graph_rag/reconcile.py +792 -0
- brain/graph_rag/relational.py +353 -0
- brain/graph_rag/retrieve.py +526 -0
- brain/graph_rag/router.py +288 -0
- brain/graph_rag/schema.py +320 -0
- brain/graph_rag/sync.py +237 -0
- brain/graph_rag/tenancy.py +43 -0
- brain/graph_rag/themes.py +501 -0
- brain/graph_rag/weighting.py +202 -0
- brain/ingest/__init__.py +1926 -0
- brain/ingest/chunker.py +249 -0
- brain/ingest/docx.py +40 -0
- brain/ingest/gmail.py +621 -0
- brain/ingest/markdown.py +37 -0
- brain/ingest/pdf.py +61 -0
- brain/ingest/stdin.py +22 -0
- brain/ingest/sub_tokens.py +91 -0
- brain/ingest/text.py +16 -0
- brain/interactions.py +205 -0
- brain/maintenance.py +355 -0
- brain/mcp_server.py +3405 -0
- brain/migrations/001_init.sql +43 -0
- brain/migrations/002_qwen3_embedding.sql +17 -0
- brain/migrations/003_vault_model.sql +41 -0
- brain/migrations/004_relax_content_hash_uniqueness.sql +18 -0
- brain/migrations/005_derived_links.sql +67 -0
- brain/migrations/006_dedup_file_by_source_path.sql +25 -0
- brain/migrations/007_email_thread_and_draft.sql +15 -0
- brain/migrations/008_gmail_thread_unique.sql +11 -0
- brain/migrations/009_chunks_weighted_tsv.sql +28 -0
- brain/migrations/010_interactions.sql +30 -0
- brain/migrations/011_documents_summary.sql +23 -0
- brain/migrations/012_graphrag.sql +171 -0
- brain/migrations/013_graphrag_communities.sql +125 -0
- brain/migrations/014_graphrag_community_summary_hash.sql +33 -0
- brain/migrations/015_interactions_graph_targets.sql +89 -0
- brain/migrations/016_index_hygiene.sql +61 -0
- brain/migrations/017_elicit.sql +30 -0
- brain/migrations/018_review_gap_signal_kinds.sql +40 -0
- brain/migrations/019_search_queries.sql +35 -0
- brain/migrations/020_link_suggestions.sql +40 -0
- brain/migrations/021_timeline_doc_date.sql +34 -0
- brain/migrations/022_link_suggestions_undirected.sql +84 -0
- brain/migrations/023_search_queries_fts_count.sql +28 -0
- brain/quartz_overrides/__init__.py +8 -0
- brain/quartz_overrides/quartz/bootstrap-cli.mjs +65 -0
- brain/quartz_overrides/quartz/build.ts +568 -0
- brain/quartz_overrides/quartz/cli/args.js +152 -0
- brain/quartz_overrides/quartz/cli/build_partial_handler.js +544 -0
- brain/quartz_overrides/quartz/cli/handlers.js +636 -0
- brain/quartz_overrides/quartz/components/CommandPalette.tsx +172 -0
- brain/quartz_overrides/quartz/components/Explorer.tsx +198 -0
- brain/quartz_overrides/quartz/components/Footer.tsx +27 -0
- brain/quartz_overrides/quartz/components/Graph.tsx +468 -0
- brain/quartz_overrides/quartz/components/PageTitle.tsx +72 -0
- brain/quartz_overrides/quartz/components/RelatedDocs.tsx +38 -0
- brain/quartz_overrides/quartz/components/Search.tsx +161 -0
- brain/quartz_overrides/quartz/components/SummaryLede.tsx +72 -0
- brain/quartz_overrides/quartz/components/index.ts +92 -0
- brain/quartz_overrides/quartz/components/pages/TagContent.tsx +272 -0
- brain/quartz_overrides/quartz/components/scripts/commandPalette.inline.ts +665 -0
- brain/quartz_overrides/quartz/components/scripts/explorer.inline.ts +768 -0
- brain/quartz_overrides/quartz/components/scripts/graph.inline.ts +2302 -0
- brain/quartz_overrides/quartz/components/scripts/relatedDocs.inline.ts +163 -0
- brain/quartz_overrides/quartz/components/scripts/search.inline.ts +1011 -0
- brain/quartz_overrides/quartz/plugins/emitters/contentIndex.ts +546 -0
- brain/quartz_overrides/quartz/plugins/transformers/codeCopy.ts +94 -0
- brain/quartz_overrides/quartz/plugins/transformers/derivedFenceMark.ts +302 -0
- brain/quartz_overrides/quartz/plugins/transformers/emailThread.ts +148 -0
- brain/quartz_overrides/quartz/plugins/transformers/emptyDoorFilter.ts +213 -0
- brain/quartz_overrides/quartz/plugins/transformers/index.ts +114 -0
- brain/quartz_overrides/quartz/plugins/transformers/linkKindMark.ts +205 -0
- brain/quartz_overrides/quartz/plugins/transformers/linkSourceTag.ts +104 -0
- brain/quartz_overrides/quartz/plugins/transformers/relativeDate.ts +100 -0
- brain/quartz_overrides/quartz/plugins/transformers/reloadSignal.ts +131 -0
- brain/quartz_overrides/quartz/processors/parse.ts +371 -0
- brain/quartz_overrides/quartz/processors/parser_cache.ts +78 -0
- brain/quartz_overrides/quartz/static/brain-logo-dark.png +0 -0
- brain/quartz_overrides/quartz/static/brain-logo-light.png +0 -0
- brain/quartz_overrides/quartz/static/codeCopy.js +196 -0
- brain/quartz_overrides/quartz/static/emailThread.js +334 -0
- brain/quartz_overrides/quartz/static/favicon.ico +0 -0
- brain/quartz_overrides/quartz/static/icon.png +0 -0
- brain/quartz_overrides/quartz/static/linkSourceTag.js +104 -0
- brain/quartz_overrides/quartz/static/relativeDate.js +142 -0
- brain/quartz_overrides/quartz/static/reload.js +168 -0
- brain/quartz_overrides/quartz/styles/brain/_article.scss +252 -0
- brain/quartz_overrides/quartz/styles/brain/_atmosphere.scss +113 -0
- brain/quartz_overrides/quartz/styles/brain/_callouts.scss +180 -0
- brain/quartz_overrides/quartz/styles/brain/_cmdk.scss +7 -0
- brain/quartz_overrides/quartz/styles/brain/_code.scss +208 -0
- brain/quartz_overrides/quartz/styles/brain/_command_palette.scss +369 -0
- brain/quartz_overrides/quartz/styles/brain/_email_thread.scss +228 -0
- brain/quartz_overrides/quartz/styles/brain/_explorer.scss +142 -0
- brain/quartz_overrides/quartz/styles/brain/_home.scss +182 -0
- brain/quartz_overrides/quartz/styles/brain/_links.scss +322 -0
- brain/quartz_overrides/quartz/styles/brain/_marginalia.scss +117 -0
- brain/quartz_overrides/quartz/styles/brain/_motion.scss +175 -0
- brain/quartz_overrides/quartz/styles/brain/_people_hub.scss +100 -0
- brain/quartz_overrides/quartz/styles/brain/_related_docs.scss +137 -0
- brain/quartz_overrides/quartz/styles/brain/_search.scss +252 -0
- brain/quartz_overrides/quartz/styles/brain/_sidebar.scss +468 -0
- brain/quartz_overrides/quartz/styles/brain/_summary_lede.scss +56 -0
- brain/quartz_overrides/quartz/styles/brain/_surface.scss +43 -0
- brain/quartz_overrides/quartz/styles/brain/_tag_content.scss +118 -0
- brain/quartz_overrides/quartz/styles/brain/_tokens.scss +197 -0
- brain/quartz_overrides/quartz/styles/brain/_typography.scss +92 -0
- brain/quartz_overrides/quartz/styles/custom.scss +89 -0
- brain/quartz_overrides/quartz/styles/graph.scss +505 -0
- brain/quartz_overrides/quartz/util/ctx.ts +92 -0
- brain/quartz_overrides/quartz/util/fastpath_manifest.ts +608 -0
- brain/quartz_overrides/quartz/util/path.ts +358 -0
- brain/quartz_overrides/quartz/util/sourceIcons.ts +55 -0
- brain/quartz_overrides/quartz.config.ts +270 -0
- brain/quartz_overrides/quartz.layout.ts +314 -0
- brain/queries.py +1188 -0
- brain/rank_fusion.py +8 -0
- brain/resurface.py +210 -0
- brain/review/__init__.py +26 -0
- brain/review/emit.py +27 -0
- brain/review/queries.py +436 -0
- brain/review/render.py +196 -0
- brain/review/scans.py +355 -0
- brain/review/weekly.py +413 -0
- brain/search.py +704 -0
- brain/set_similarity.py +15 -0
- brain/setup.py +1205 -0
- brain/tags.py +56 -0
- brain/templates/Caddyfile.j2 +9 -0
- brain/templates/__init__.py +1 -0
- brain/templates/bin/__init__.py +1 -0
- brain/templates/bin/_brain-brief-fg.sh +25 -0
- brain/templates/bin/_brain-build-fg.sh +53 -0
- brain/templates/bin/_brain-watcher-fg.sh +65 -0
- brain/templates/bin/brain-down.sh +89 -0
- brain/templates/bin/brain-status.sh +83 -0
- brain/templates/bin/brain-up.sh +221 -0
- brain/templates/docker/age/Dockerfile +79 -0
- brain/templates/docker-compose.stock.yml.j2 +26 -0
- brain/templates/docker-compose.yml.j2 +34 -0
- brain/templates/env.example +190 -0
- brain/templates/launchd/__init__.py +1 -0
- brain/templates/launchd/com.brain.brief.plist.j2 +45 -0
- brain/templates/launchd/com.brain.build.plist.j2 +46 -0
- brain/templates/launchd/com.brain.watcher.plist.j2 +46 -0
- brain/templates/skill/SKILL.md +63 -0
- brain/templates/skill/__init__.py +1 -0
- brain/timeline.py +834 -0
- brain/todo.py +124 -0
- brain/uninstall.py +185 -0
- brain/vault/__init__.py +115 -0
- brain/vault/_atomic.py +25 -0
- brain/vault/daily_index.py +228 -0
- brain/vault/derived_links/__init__.py +50 -0
- brain/vault/derived_links/directory.py +683 -0
- brain/vault/derived_links/fence.py +408 -0
- brain/vault/derived_links/gws.py +64 -0
- brain/vault/derived_links/participants.py +143 -0
- brain/vault/derived_links/pass_runner.py +362 -0
- brain/vault/derived_links/rules.py +137 -0
- brain/vault/export.py +683 -0
- brain/vault/frontmatter.py +165 -0
- brain/vault/graph.py +620 -0
- brain/vault/graph_format.py +388 -0
- brain/vault/link_rewrite.py +235 -0
- brain/vault/links.py +260 -0
- brain/vault/note_builder.py +211 -0
- brain/vault/paths.py +55 -0
- brain/vault/quartz_overlay.py +236 -0
- brain/vault/rename.py +591 -0
- brain/vault/resolver.py +304 -0
- brain/vault/slug.py +127 -0
- brain/vault/sync.py +1513 -0
- brain/vault/sync_summaries.py +264 -0
- brain/vault/templates.py +145 -0
- brain/vault/watch.py +1052 -0
- brain/wiki/__init__.py +6 -0
- brain/wiki/_github_slugger.py +76 -0
- brain/wiki/_person_name.py +314 -0
- brain/wiki/build_homepage.py +541 -0
- brain/wiki/build_partial.py +273 -0
- brain/wiki/build_people.py +934 -0
- brain/wiki/build_related.py +758 -0
- brain/wiki/build_swap.py +585 -0
- brain/wiki/build_watcher.py +975 -0
- brain/wiki/edit_classifier.py +215 -0
- brain/wiki/errors.py +10 -0
- brain/wiki/fastpath_manifest.py +475 -0
- brain/wiki/fastpath_state.py +174 -0
- brain/wiki/install.py +296 -0
- brain/wiki/slug.py +111 -0
- secondbrain_py-0.2.1.dist-info/METADATA +195 -0
- secondbrain_py-0.2.1.dist-info/RECORD +273 -0
- secondbrain_py-0.2.1.dist-info/WHEEL +5 -0
- secondbrain_py-0.2.1.dist-info/entry_points.txt +11 -0
- secondbrain_py-0.2.1.dist-info/licenses/LICENSE +21 -0
- secondbrain_py-0.2.1.dist-info/top_level.txt +1 -0
brain/ingest/__init__.py
ADDED
|
@@ -0,0 +1,1926 @@
|
|
|
1
|
+
"""Ingest pipeline: extract → chunk → embed → store."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import hashlib
|
|
5
|
+
import json
|
|
6
|
+
import logging
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from datetime import UTC, datetime, timedelta
|
|
9
|
+
from email.utils import parsedate_to_datetime
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import TYPE_CHECKING, Any, Protocol
|
|
12
|
+
|
|
13
|
+
import psycopg
|
|
14
|
+
from pgvector.psycopg import register_vector # noqa: F401 (ensures adapter loaded)
|
|
15
|
+
|
|
16
|
+
from brain.errors import EnrichmentError, IngestAmbiguousSource, OllamaUnavailable
|
|
17
|
+
from brain.tags import normalize_tags
|
|
18
|
+
|
|
19
|
+
if TYPE_CHECKING:
|
|
20
|
+
from brain.enrichment import OllamaEnricher
|
|
21
|
+
from brain.graph_rag.sync import GraphSyncer
|
|
22
|
+
from brain.vault.derived_links.directory import (
|
|
23
|
+
DirectoryStore,
|
|
24
|
+
GwsRunner,
|
|
25
|
+
refresh_calendar,
|
|
26
|
+
refresh_contacts,
|
|
27
|
+
)
|
|
28
|
+
from brain.vault.derived_links.participants import (
|
|
29
|
+
extract_gmail_addresses,
|
|
30
|
+
extract_krisp_speakers,
|
|
31
|
+
)
|
|
32
|
+
from brain.vault.export import regenerate_vault_file
|
|
33
|
+
|
|
34
|
+
from .chunker import Chunk, chunk_text
|
|
35
|
+
from .sub_tokens import extract_sub_tokens
|
|
36
|
+
|
|
37
|
+
_logger = logging.getLogger(__name__)
|
|
38
|
+
|
|
39
|
+
# Contacts refresh is rate-limited to once per 24 hours. Krisp ingest is the
|
|
40
|
+
# trigger; without this gate, every transcript would re-fetch the full Google
|
|
41
|
+
# People page.
|
|
42
|
+
_CONTACTS_REFRESH_INTERVAL = timedelta(hours=24)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class Embedder(Protocol):
|
|
46
|
+
"""Narrow interface for embedding clients used by the ingest pipeline.
|
|
47
|
+
|
|
48
|
+
``dim`` is the embedder's native output dimension. Schema-wiring code
|
|
49
|
+
(``db.ensure_embedding_column``, ``queries.finalize_embedding_index``)
|
|
50
|
+
reads it to keep the ``chunks.embedding`` column in lockstep with the
|
|
51
|
+
active backend, so callers stay backend-agnostic.
|
|
52
|
+
"""
|
|
53
|
+
|
|
54
|
+
dim: int
|
|
55
|
+
|
|
56
|
+
def embed(
|
|
57
|
+
self, texts: list[str], *, input_type: str = "document"
|
|
58
|
+
) -> list[list[float]]: ...
|
|
59
|
+
|
|
60
|
+
def count_tokens(self, text: str) -> int: ...
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
@dataclass
|
|
64
|
+
class ExtractedDoc:
|
|
65
|
+
"""A document produced by an extractor and ready to be ingested."""
|
|
66
|
+
|
|
67
|
+
title: str
|
|
68
|
+
content: str
|
|
69
|
+
content_type: str
|
|
70
|
+
source_path: str | None
|
|
71
|
+
metadata: dict[str, Any]
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
@dataclass
|
|
75
|
+
class IngestResult:
|
|
76
|
+
"""Outcome of :func:`ingest_document`.
|
|
77
|
+
|
|
78
|
+
``body_changed`` is ``True`` iff ``existing_hash != incoming_hash`` at the
|
|
79
|
+
moment the in-place UPDATE was applied. Concretely:
|
|
80
|
+
|
|
81
|
+
- New document (``created=True``): always ``True``.
|
|
82
|
+
- In-place UPDATE (``created=False``) where the stored content_hash
|
|
83
|
+
differs from the incoming hash: ``True`` — body was rewritten.
|
|
84
|
+
- In-place UPDATE where hashes match (same body, ``--force`` or changed
|
|
85
|
+
tags): ``False`` — no body change even though chunks were mechanically
|
|
86
|
+
rebuilt.
|
|
87
|
+
- No-op short-circuit (same hash, no ``--force``): ``False``.
|
|
88
|
+
|
|
89
|
+
Note: ``_update_doc_in_place`` always rebuilds chunks (DELETE + INSERT)
|
|
90
|
+
regardless of ``body_changed`` — chunk row UUIDs change on every forced
|
|
91
|
+
re-ingest, but ``body_changed`` reports the *content-hash identity*, not
|
|
92
|
+
the chunk-row machinery. The mirror trigger in :func:`ingest_document`
|
|
93
|
+
reads ``body_changed OR force`` so an in-place update with only tag changes
|
|
94
|
+
still propagates to disk.
|
|
95
|
+
"""
|
|
96
|
+
|
|
97
|
+
document_id: str | None
|
|
98
|
+
created: bool
|
|
99
|
+
body_changed: bool = False
|
|
100
|
+
# Wave G1-c: set ONLY on the force content-hash-fallback path
|
|
101
|
+
# (:func:`_ingest_within_transaction`), where an old ``documents`` row at
|
|
102
|
+
# the same content hash is DELETEd and a fresh row INSERTed under a NEW
|
|
103
|
+
# uuid. The old uuid's relational graph rows cascade away with the row, but
|
|
104
|
+
# its AGE ``Document`` vertex (and any persons it orphaned) do NOT — so
|
|
105
|
+
# :func:`ingest_document` calls ``graph_syncer.remove(replaced_document_id)``
|
|
106
|
+
# post-commit to drop the stale graph presence. ``None`` on every other
|
|
107
|
+
# path (new insert, in-place update, no-op skip).
|
|
108
|
+
replaced_document_id: str | None = None
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
@dataclass
|
|
112
|
+
class UpdateResult:
|
|
113
|
+
"""Outcome of :func:`update_document`.
|
|
114
|
+
|
|
115
|
+
``fields_changed`` lists the document columns that were actually mutated
|
|
116
|
+
(subset of ``{"title", "content", "content_type", "metadata", "tags"}``).
|
|
117
|
+
``rechunked`` is ``True`` iff the body was replaced and chunks were
|
|
118
|
+
re-embedded.
|
|
119
|
+
"""
|
|
120
|
+
|
|
121
|
+
document_id: str
|
|
122
|
+
fields_changed: list[str]
|
|
123
|
+
rechunked: bool
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _content_hash(text: str) -> str:
|
|
127
|
+
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _chunk_search_metadata(title: str, tags: list[str]) -> tuple[str, str]:
|
|
131
|
+
"""Compute the per-document ``(title_text, tags_text)`` chunk columns.
|
|
132
|
+
|
|
133
|
+
Migration 009 splits chunks.tsv into a weighted multi-field tsvector — the
|
|
134
|
+
title goes under weight ``A`` and the tags under weight ``B``. Both are
|
|
135
|
+
denormalized onto every chunk row so the FTS rewrite stays a single
|
|
136
|
+
GIN-indexable expression. This helper centralizes the projection so all
|
|
137
|
+
four ``INSERT INTO chunks`` sites + the ``sync_chunk_search_metadata``
|
|
138
|
+
helper in :mod:`brain.queries` agree on the exact textual form.
|
|
139
|
+
|
|
140
|
+
``tags_text`` mirrors the SQL backfill pattern
|
|
141
|
+
``array_to_string(d.tags, ' ')`` — space-joined, empty string when the
|
|
142
|
+
tag list is empty (NOT ``None``; we never write SQL NULL here so the
|
|
143
|
+
coalesce inside the generated tsv stays a no-op rather than a noisy
|
|
144
|
+
cast).
|
|
145
|
+
"""
|
|
146
|
+
return title, " ".join(tags) if tags else ""
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _is_gmail_thread_doc(doc: ExtractedDoc, source_kind: str) -> bool:
|
|
150
|
+
"""Return True iff ``doc`` is a P2.1 merged-thread Gmail doc.
|
|
151
|
+
|
|
152
|
+
The marker shape — ``source_kind == "gmail"`` AND
|
|
153
|
+
``content_type == "email_thread"`` AND ``metadata.thread_id`` populated
|
|
154
|
+
with a non-empty string — is the same triple that the partial unique
|
|
155
|
+
index ``uq_documents_gmail_thread`` (migration 008) constrains. Anything
|
|
156
|
+
else (legacy ``content_type == "email"`` per-message rows, manual
|
|
157
|
+
``email_thread`` docs without a thread_id) falls through to the legacy
|
|
158
|
+
content_hash dedup path. Centralising the check here keeps the call
|
|
159
|
+
sites in :func:`ingest_document` and :func:`_ingest_within_transaction`
|
|
160
|
+
in lockstep — divergence between them would silently break the upsert.
|
|
161
|
+
"""
|
|
162
|
+
thread_id = doc.metadata.get("thread_id")
|
|
163
|
+
return (
|
|
164
|
+
source_kind == "gmail"
|
|
165
|
+
and doc.content_type == "email_thread"
|
|
166
|
+
and isinstance(thread_id, str)
|
|
167
|
+
and bool(thread_id)
|
|
168
|
+
)
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
# Metadata keys promoted into typed columns on ``documents``. The mapping is
|
|
172
|
+
# intentionally narrow: every key here must correspond to a column added by
|
|
173
|
+
# migration 007. Anything outside this set stays in the JSONB ``metadata``
|
|
174
|
+
# blob, untouched.
|
|
175
|
+
_PROMOTED_COLUMNS: tuple[str, ...] = (
|
|
176
|
+
"thread_id",
|
|
177
|
+
"rfc_message_id",
|
|
178
|
+
"in_reply_to",
|
|
179
|
+
"sent_at",
|
|
180
|
+
"participants",
|
|
181
|
+
"duration_min",
|
|
182
|
+
)
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def _parse_sent_at(raw: Any) -> datetime | None:
|
|
186
|
+
"""Parse a date string into a TZ-aware UTC ``datetime``.
|
|
187
|
+
|
|
188
|
+
Accepts both RFC 2822 (Gmail ``Date:`` header style — e.g.
|
|
189
|
+
``"Tue, 04 May 2026 14:23:01 -0400"``) and ISO 8601 (Krisp-style — e.g.
|
|
190
|
+
``"2026-05-04T14:23:01+00:00"``) inputs. Returns ``None`` for any input
|
|
191
|
+
we can't parse — callers log + skip the column rather than crashing the
|
|
192
|
+
ingest. Naive datetimes are treated as UTC.
|
|
193
|
+
"""
|
|
194
|
+
if not isinstance(raw, str) or not raw.strip():
|
|
195
|
+
return None
|
|
196
|
+
|
|
197
|
+
parsed: datetime | None = None
|
|
198
|
+
# RFC 2822 first — ``parsedate_to_datetime`` is strict-but-tolerant and
|
|
199
|
+
# is the format Gmail's ``Date:`` header uses, so it's the more common
|
|
200
|
+
# case for this codebase.
|
|
201
|
+
try:
|
|
202
|
+
parsed = parsedate_to_datetime(raw)
|
|
203
|
+
except (TypeError, ValueError):
|
|
204
|
+
parsed = None
|
|
205
|
+
if parsed is None:
|
|
206
|
+
try:
|
|
207
|
+
# ``fromisoformat`` accepts the trailing-Z form in 3.11+.
|
|
208
|
+
parsed = datetime.fromisoformat(raw.replace("Z", "+00:00"))
|
|
209
|
+
except ValueError:
|
|
210
|
+
return None
|
|
211
|
+
|
|
212
|
+
if parsed.tzinfo is None:
|
|
213
|
+
parsed = parsed.replace(tzinfo=UTC)
|
|
214
|
+
return parsed.astimezone(UTC)
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def _promote_metadata_to_columns(metadata: dict[str, Any]) -> dict[str, Any]:
|
|
218
|
+
"""Project a metadata blob onto the typed ``documents`` columns.
|
|
219
|
+
|
|
220
|
+
Returns a ``{column_name: value}`` dict — keys are limited to the columns
|
|
221
|
+
in :data:`_PROMOTED_COLUMNS`. Missing-or-``None`` metadata keys are
|
|
222
|
+
omitted (callers never need to write a NULL explicitly — the column
|
|
223
|
+
default of NULL handles it on INSERT, and on UPDATE we just leave the
|
|
224
|
+
column alone). Malformed values are logged at WARNING and skipped so a
|
|
225
|
+
bad header never blocks an ingest.
|
|
226
|
+
|
|
227
|
+
Type-coercion rules:
|
|
228
|
+
|
|
229
|
+
- ``thread_id`` / ``rfc_message_id`` / ``in_reply_to`` — accept ``str``;
|
|
230
|
+
anything else is logged + skipped.
|
|
231
|
+
- ``date`` (RFC 2822 or ISO 8601 string) → ``sent_at`` (TZ-aware UTC).
|
|
232
|
+
Unparseable strings log + skip; we never store a partially-parsed
|
|
233
|
+
datetime.
|
|
234
|
+
- ``participants`` — accept ``list[str]`` (Krisp speakers, Gmail headers
|
|
235
|
+
already-parsed by an upstream extractor). Non-list / mixed-type lists
|
|
236
|
+
log + skip rather than silently coercing.
|
|
237
|
+
- ``duration_min`` — accept ``int``, ``float``, or a numeric ``str``.
|
|
238
|
+
Floats — and float-valued strings like ``"42.7"`` — round down via
|
|
239
|
+
``int(float(...))``; ``bool`` and non-numeric values log + skip.
|
|
240
|
+
"""
|
|
241
|
+
out: dict[str, Any] = {}
|
|
242
|
+
|
|
243
|
+
for key in ("thread_id", "rfc_message_id", "in_reply_to"):
|
|
244
|
+
raw = metadata.get(key)
|
|
245
|
+
if raw is None:
|
|
246
|
+
continue
|
|
247
|
+
if isinstance(raw, str):
|
|
248
|
+
out[key] = raw
|
|
249
|
+
else:
|
|
250
|
+
_logger.warning(
|
|
251
|
+
"metadata.%s expected str, got %s; skipping column promotion",
|
|
252
|
+
key,
|
|
253
|
+
type(raw).__name__,
|
|
254
|
+
)
|
|
255
|
+
|
|
256
|
+
raw_date = metadata.get("date")
|
|
257
|
+
if raw_date is not None:
|
|
258
|
+
sent_at = _parse_sent_at(raw_date)
|
|
259
|
+
if sent_at is not None:
|
|
260
|
+
out["sent_at"] = sent_at
|
|
261
|
+
else:
|
|
262
|
+
_logger.warning(
|
|
263
|
+
"metadata.date is not parseable (%r); leaving sent_at NULL",
|
|
264
|
+
raw_date,
|
|
265
|
+
)
|
|
266
|
+
|
|
267
|
+
raw_participants = metadata.get("participants")
|
|
268
|
+
if raw_participants is not None:
|
|
269
|
+
if isinstance(raw_participants, list) and all(
|
|
270
|
+
isinstance(p, str) for p in raw_participants
|
|
271
|
+
):
|
|
272
|
+
out["participants"] = list(raw_participants)
|
|
273
|
+
else:
|
|
274
|
+
_logger.warning(
|
|
275
|
+
"metadata.participants expected list[str], got %r; "
|
|
276
|
+
"skipping column promotion",
|
|
277
|
+
type(raw_participants).__name__,
|
|
278
|
+
)
|
|
279
|
+
|
|
280
|
+
raw_duration = metadata.get("duration_min")
|
|
281
|
+
if raw_duration is not None:
|
|
282
|
+
try:
|
|
283
|
+
# ``bool`` is a subclass of ``int`` in Python — reject it
|
|
284
|
+
# explicitly so ``True`` doesn't end up as ``1`` in the column.
|
|
285
|
+
if isinstance(raw_duration, bool):
|
|
286
|
+
raise TypeError("bool is not a valid duration")
|
|
287
|
+
# ``int(float(...))`` so numeric strings with a fractional part
|
|
288
|
+
# ("42.7", Krisp's occasional float-minute serialization) round
|
|
289
|
+
# down instead of raising ValueError under a bare ``int("42.7")``.
|
|
290
|
+
out["duration_min"] = int(float(raw_duration))
|
|
291
|
+
except (TypeError, ValueError):
|
|
292
|
+
_logger.warning(
|
|
293
|
+
"metadata.duration_min expected int, got %r; "
|
|
294
|
+
"skipping column promotion",
|
|
295
|
+
raw_duration,
|
|
296
|
+
)
|
|
297
|
+
|
|
298
|
+
return out
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def _upsert_source(
|
|
302
|
+
conn: psycopg.Connection,
|
|
303
|
+
*,
|
|
304
|
+
kind: str,
|
|
305
|
+
external_id: str | None,
|
|
306
|
+
metadata: dict[str, Any],
|
|
307
|
+
) -> str | None:
|
|
308
|
+
"""Return an existing source row id, or insert a new one. Returns None for
|
|
309
|
+
purely manual ingests with no external id and no metadata.
|
|
310
|
+
|
|
311
|
+
Wave perf-T3 (F6): the common (non-NULL ``external_id``) path is a single
|
|
312
|
+
``INSERT … ON CONFLICT (kind, external_id)`` round-trip instead of a
|
|
313
|
+
SELECT-then-INSERT pair. The ``DO UPDATE SET metadata = sources.metadata``
|
|
314
|
+
clause is a deliberate no-op self-assignment — it forces ``RETURNING id`` to
|
|
315
|
+
yield the row even on conflict (``DO NOTHING`` returns no row) while
|
|
316
|
+
preserving the existing row's metadata, exactly matching the legacy
|
|
317
|
+
behavior, which returned the existing id WITHOUT refreshing metadata.
|
|
318
|
+
|
|
319
|
+
The ``UNIQUE (kind, external_id)`` constraint (migration 001) treats NULLs
|
|
320
|
+
as distinct, so an ``ON CONFLICT`` target cannot match a NULL
|
|
321
|
+
``external_id``. The rare manual-stdin-with-metadata case
|
|
322
|
+
(``external_id is None`` but ``metadata`` non-empty) keeps the explicit
|
|
323
|
+
lookup-then-insert so it still reuses an existing NULL-external_id row.
|
|
324
|
+
"""
|
|
325
|
+
if external_id is None and not metadata:
|
|
326
|
+
return None
|
|
327
|
+
if external_id is None:
|
|
328
|
+
row = conn.execute(
|
|
329
|
+
"SELECT id FROM sources WHERE kind=%s AND external_id IS NULL",
|
|
330
|
+
(kind,),
|
|
331
|
+
).fetchone()
|
|
332
|
+
if row:
|
|
333
|
+
return str(row[0])
|
|
334
|
+
new = conn.execute(
|
|
335
|
+
"INSERT INTO sources (kind, external_id, metadata) "
|
|
336
|
+
"VALUES (%s, NULL, %s) RETURNING id",
|
|
337
|
+
(kind, json.dumps(metadata)),
|
|
338
|
+
).fetchone()
|
|
339
|
+
assert new is not None # RETURNING id always yields a row
|
|
340
|
+
return str(new[0])
|
|
341
|
+
row = conn.execute(
|
|
342
|
+
"INSERT INTO sources (kind, external_id, metadata) VALUES (%s, %s, %s) "
|
|
343
|
+
"ON CONFLICT (kind, external_id) DO UPDATE SET metadata = sources.metadata "
|
|
344
|
+
"RETURNING id",
|
|
345
|
+
(kind, external_id, json.dumps(metadata)),
|
|
346
|
+
).fetchone()
|
|
347
|
+
assert row is not None # ON CONFLICT DO UPDATE always yields a row
|
|
348
|
+
return str(row[0])
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def _embed_chunks(
|
|
352
|
+
embedder: Embedder, texts: list[str]
|
|
353
|
+
) -> list[list[float] | None]:
|
|
354
|
+
"""Embed ``texts``, or return NULL placeholders under a no-vector backend.
|
|
355
|
+
|
|
356
|
+
Under the FTS-only ``none`` backend (:class:`brain.embeddings.NullEmbedder`,
|
|
357
|
+
``produces_embeddings = False``) the pipeline must store NULL embeddings
|
|
358
|
+
rather than call ``embed()`` (which raises :class:`~brain.errors.EmbedError`).
|
|
359
|
+
Duck-typed via ``getattr`` so the real backends (Arctic / Qwen3 / Voyage) —
|
|
360
|
+
which never declare the flag — embed normally with zero changes. Returns one
|
|
361
|
+
entry per input text: a real vector, or ``None`` (bound as SQL NULL by
|
|
362
|
+
:func:`_insert_chunks`; the column is nullable pre-finalize). An empty
|
|
363
|
+
``texts`` makes no embed call and returns ``[]``.
|
|
364
|
+
"""
|
|
365
|
+
if not getattr(embedder, "produces_embeddings", True):
|
|
366
|
+
return [None] * len(texts)
|
|
367
|
+
# Widen the concrete ``list[list[float]]`` to the nullable element type via
|
|
368
|
+
# ``extend`` (Iterable is covariant, so this stays mypy-clean) — the real
|
|
369
|
+
# backends only ever contribute non-NULL vectors here.
|
|
370
|
+
embeddings: list[list[float] | None] = []
|
|
371
|
+
embeddings.extend(embedder.embed(texts, input_type="document"))
|
|
372
|
+
return embeddings
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
def _insert_chunks(
|
|
376
|
+
conn: psycopg.Connection,
|
|
377
|
+
*,
|
|
378
|
+
document_id: str,
|
|
379
|
+
chunks: list[Chunk],
|
|
380
|
+
embeddings: list[list[float] | None],
|
|
381
|
+
title_text: str,
|
|
382
|
+
tags_text: str,
|
|
383
|
+
) -> None:
|
|
384
|
+
"""Bulk-insert chunk rows for ``document_id`` in a single round-trip.
|
|
385
|
+
|
|
386
|
+
Wave perf-T3 (F2/F7): replaces the per-chunk ``conn.execute`` loop with one
|
|
387
|
+
``executemany`` over a pre-built row list. ``extract_sub_tokens`` is computed
|
|
388
|
+
while building the list (before any DB round-trip) instead of interleaved
|
|
389
|
+
with N inserts. The ``title_text`` / ``tags_text`` / ``search_extras``
|
|
390
|
+
columns feed the migration-009 weighted multi-field tsv exactly as the prior
|
|
391
|
+
loop did, so the stored rows are byte-for-byte identical. ``embeddings`` must
|
|
392
|
+
be precomputed by the caller (outside the transaction). No-op on empty
|
|
393
|
+
``chunks``.
|
|
394
|
+
"""
|
|
395
|
+
if not chunks:
|
|
396
|
+
return
|
|
397
|
+
rows = [
|
|
398
|
+
(
|
|
399
|
+
document_id,
|
|
400
|
+
c.index,
|
|
401
|
+
c.content,
|
|
402
|
+
emb,
|
|
403
|
+
title_text,
|
|
404
|
+
tags_text,
|
|
405
|
+
extract_sub_tokens(c.content),
|
|
406
|
+
)
|
|
407
|
+
for c, emb in zip(chunks, embeddings, strict=True)
|
|
408
|
+
]
|
|
409
|
+
# psycopg3 exposes ``executemany`` on the cursor, not the connection. The
|
|
410
|
+
# cursor runs inside the caller's open transaction (same connection), so
|
|
411
|
+
# the bulk insert stays atomic with the surrounding document write.
|
|
412
|
+
with conn.cursor() as cur:
|
|
413
|
+
cur.executemany(
|
|
414
|
+
"INSERT INTO chunks (document_id, chunk_index, content, embedding, "
|
|
415
|
+
"title_text, tags_text, search_extras) "
|
|
416
|
+
"VALUES (%s, %s, %s, %s, %s, %s, %s)",
|
|
417
|
+
rows,
|
|
418
|
+
)
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
def ingest_document(
|
|
422
|
+
conn: psycopg.Connection,
|
|
423
|
+
*,
|
|
424
|
+
embedder: Embedder,
|
|
425
|
+
doc: ExtractedDoc,
|
|
426
|
+
source_kind: str | None = None,
|
|
427
|
+
source_external_id: str | None = None,
|
|
428
|
+
source_metadata: dict[str, Any] | None = None,
|
|
429
|
+
tags: list[str] | None = None,
|
|
430
|
+
force: bool = False,
|
|
431
|
+
gws_runner: GwsRunner | None = None,
|
|
432
|
+
vault_root: Path | None = None,
|
|
433
|
+
draft: bool = False,
|
|
434
|
+
enricher: OllamaEnricher | None = None,
|
|
435
|
+
enrich: bool = True,
|
|
436
|
+
enrich_min_tokens: int = 50,
|
|
437
|
+
graph_syncer: GraphSyncer | None = None,
|
|
438
|
+
) -> IngestResult:
|
|
439
|
+
"""Ingest a single extracted document.
|
|
440
|
+
|
|
441
|
+
Dedup rules (evaluated in order):
|
|
442
|
+
|
|
443
|
+
1. **Gmail thread** — deduped by ``(thread_id, content_type='email_thread')``.
|
|
444
|
+
Re-ingest of the same thread UPDATEs the existing row in place (UUID
|
|
445
|
+
stable; links / derived_links / unresolved_links preserved).
|
|
446
|
+
|
|
447
|
+
2. **File-based** (``doc.source_path`` is not ``None``) — deduped by
|
|
448
|
+
``source_path`` (``WHERE source_path=%s AND kind='ingested'``). When a
|
|
449
|
+
match is found the row is UPDATEd in place: the document UUID is
|
|
450
|
+
preserved, tags are **union**-merged (existing ∪ incoming, insertion
|
|
451
|
+
order), and ``links`` / ``derived_links`` / ``unresolved_links``
|
|
452
|
+
referencing the UUID survive. Chunks are always rebuilt. When no
|
|
453
|
+
source_path match exists the doc is INSERTed as a new row directly —
|
|
454
|
+
migration 006 narrows the content_hash UNIQUE constraint to stdin-only
|
|
455
|
+
rows, so the INSERT cannot collide with another file at a different path.
|
|
456
|
+
``force=True`` forces the UPDATE even when the content_hash is unchanged.
|
|
457
|
+
|
|
458
|
+
3. **Sourced stdin** (``source_path`` is ``None``, ``source_external_id``
|
|
459
|
+
is not ``None`` — Krisp, Slack, ...) — deduped by
|
|
460
|
+
``(source_kind, source_external_id)`` via a JOIN on ``sources``. When a
|
|
461
|
+
match is found the row is UPDATEd in place (same UUID / tag union /
|
|
462
|
+
link-preservation semantics as the file branch). When no sourced row
|
|
463
|
+
exists, control falls through to rule 4.
|
|
464
|
+
|
|
465
|
+
4. **Content-hash fallback** — reached only when ``source_path is None``
|
|
466
|
+
(stdin ingests). Deduped by SHA-256 of ``doc.content`` scoped to
|
|
467
|
+
``kind='ingested' AND source_path IS NULL``. Same-hash is a no-op;
|
|
468
|
+
``force=True`` does DELETE + INSERT (new UUID).
|
|
469
|
+
|
|
470
|
+
Tags are **union**-merged on all UPDATE paths: existing curated tags are
|
|
471
|
+
preserved; incoming tags are added. The union is computed via
|
|
472
|
+
``normalize_tags([*existing, *incoming])`` which is idempotent (casefold +
|
|
473
|
+
hyphenate) and preserves first-seen insertion order.
|
|
474
|
+
|
|
475
|
+
Sources are deduped by ``(kind, external_id)``. A repeat ingest pointing
|
|
476
|
+
at the same external id reuses the existing source row.
|
|
477
|
+
|
|
478
|
+
Source defaults: file-based ingests (``doc.source_path`` set) default to
|
|
479
|
+
``source_kind="manual"`` with ``source_external_id=doc.source_path`` when
|
|
480
|
+
the caller does not pass them, so the resulting document carries a
|
|
481
|
+
``source_id`` pointing at a ``sources`` row with ``kind="manual"``.
|
|
482
|
+
Without this, downstream consumers (vault-export frontmatter, the graph
|
|
483
|
+
filter chips) lose the manual-source signal. Explicit ``source_kind`` /
|
|
484
|
+
``source_external_id`` arguments always win, so a file-based ingest can
|
|
485
|
+
still be tagged as e.g. ``krisp`` if needed. Stdin ingests must pass
|
|
486
|
+
``source_kind`` explicitly — they get no default.
|
|
487
|
+
|
|
488
|
+
Source-specific side effects split by cost (Task 2.11): the Gmail directory
|
|
489
|
+
upsert runs IN-TRANSACTION via :func:`_run_source_hooks` (fast local write
|
|
490
|
+
with an atomic-rollback contract), while the Krisp calendar / contacts
|
|
491
|
+
refresh runs POST-COMMIT via :func:`_run_post_commit_source_hooks` because it
|
|
492
|
+
shells out to ``gws`` (multi-second) and must not hold the write
|
|
493
|
+
transaction. The ``gws_runner`` argument is only consulted by the Krisp hook;
|
|
494
|
+
passing ``None`` skips the calendar / contacts refresh with a logged warning
|
|
495
|
+
so callers without a runner wired (early CLI paths, tests) still succeed.
|
|
496
|
+
|
|
497
|
+
Vault mirror: when ``vault_root`` is supplied AND the call actually wrote
|
|
498
|
+
a row (``result.created`` or ``force``), the corresponding mirror file
|
|
499
|
+
under ``vault_root / _ingested/<source>/...md`` is regenerated via
|
|
500
|
+
:func:`brain.vault.export.regenerate_vault_file`. The call runs OUTSIDE
|
|
501
|
+
the DB transaction — a filesystem failure is logged at WARNING and
|
|
502
|
+
swallowed so a transient mirror error never rolls back a successful
|
|
503
|
+
ingest. Recovery is via ``brain vault export --force``. Library callers
|
|
504
|
+
(tests, internal pipelines) that don't care about the mirror omit
|
|
505
|
+
``vault_root`` and get the legacy DB-only behavior unchanged. Named
|
|
506
|
+
``vault_root`` to distinguish from the per-document
|
|
507
|
+
``documents.vault_path`` relative path.
|
|
508
|
+
|
|
509
|
+
Wave Q1-D enrichment: when ``enrich=True`` (default) AND ``enricher`` is
|
|
510
|
+
not None, a post-ingest hook generates a 2-3 sentence summary via local
|
|
511
|
+
Ollama and writes it to ``documents.summary`` inside the same
|
|
512
|
+
transaction. Skip rules: caller passes ``enrich=False``, content is
|
|
513
|
+
shorter than ``enrich_min_tokens`` tokens, ``documents.summary`` is
|
|
514
|
+
already populated with the same ``content_hash``, or Ollama is
|
|
515
|
+
unavailable (logged at WARNING — the ingest still commits;
|
|
516
|
+
``brain enrich --backfill`` picks the row up later). Passing
|
|
517
|
+
``enricher=None`` with ``enrich=True`` is a no-op with a debug log
|
|
518
|
+
(library callers / tests that don't care about enrichment leave it
|
|
519
|
+
None).
|
|
520
|
+
|
|
521
|
+
Wave G1-c graph sync: when ``graph_syncer`` is supplied AND the call
|
|
522
|
+
actually wrote a row (``created`` / ``body_changed`` / ``force``), the
|
|
523
|
+
document's people-aspect graph is reconciled post-commit via
|
|
524
|
+
:meth:`brain.graph_rag.sync.GraphSyncer.reconcile` (reusing ``conn``,
|
|
525
|
+
OUTSIDE the transaction — like the vault mirror). It is best-effort and
|
|
526
|
+
never raises: a disabled flag, an AGE-absent DB, or any sync error is a
|
|
527
|
+
logged no-op that leaves the committed ingest intact. ``None`` (library
|
|
528
|
+
callers / tests) skips graph sync entirely.
|
|
529
|
+
"""
|
|
530
|
+
if doc.source_path is not None:
|
|
531
|
+
if source_kind is None:
|
|
532
|
+
source_kind = "manual"
|
|
533
|
+
if source_external_id is None:
|
|
534
|
+
source_external_id = doc.source_path
|
|
535
|
+
if source_kind is None:
|
|
536
|
+
raise ValueError(
|
|
537
|
+
"source_kind is required when doc.source_path is None"
|
|
538
|
+
)
|
|
539
|
+
|
|
540
|
+
# Gmail-thread upsert (P2.2): a merged-thread doc keys on the stable
|
|
541
|
+
# Gmail ``threadId`` rather than a per-message ``messageId``. Mirroring
|
|
542
|
+
# that invariant onto the ``sources`` row keeps source dedup aligned
|
|
543
|
+
# with the new partial unique index ``uq_documents_gmail_thread`` —
|
|
544
|
+
# repeated thread-batched ingests reuse the same ``sources`` row instead
|
|
545
|
+
# of accumulating one per re-ingest. The override only fires when the
|
|
546
|
+
# caller actually produced a thread doc (``content_type='email_thread'``
|
|
547
|
+
# AND ``metadata.thread_id`` populated); legacy per-message gmail rows
|
|
548
|
+
# still pass message_id as ``source_external_id`` and are unaffected.
|
|
549
|
+
if _is_gmail_thread_doc(doc, source_kind):
|
|
550
|
+
thread_id = doc.metadata.get("thread_id")
|
|
551
|
+
# _is_gmail_thread_doc guarantees thread_id is a non-empty str.
|
|
552
|
+
assert isinstance(thread_id, str) and thread_id
|
|
553
|
+
source_external_id = thread_id
|
|
554
|
+
|
|
555
|
+
h = _content_hash(doc.content)
|
|
556
|
+
tags = tags or []
|
|
557
|
+
source_metadata = source_metadata or {}
|
|
558
|
+
|
|
559
|
+
result = _ingest_within_transaction(
|
|
560
|
+
conn,
|
|
561
|
+
embedder=embedder,
|
|
562
|
+
doc=doc,
|
|
563
|
+
source_kind=source_kind,
|
|
564
|
+
source_external_id=source_external_id,
|
|
565
|
+
source_metadata=source_metadata,
|
|
566
|
+
tags=tags,
|
|
567
|
+
force=force,
|
|
568
|
+
gws_runner=gws_runner,
|
|
569
|
+
content_hash=h,
|
|
570
|
+
draft=draft,
|
|
571
|
+
enricher=enricher,
|
|
572
|
+
enrich=enrich,
|
|
573
|
+
enrich_min_tokens=enrich_min_tokens,
|
|
574
|
+
)
|
|
575
|
+
|
|
576
|
+
# Mirror writes happen OUTSIDE the transaction so a filesystem error
|
|
577
|
+
# cannot roll back the DB ingest (CLAUDE.md: prefer recoverable drift
|
|
578
|
+
# over an aborted ingest — drift is exactly what `brain vault export`
|
|
579
|
+
# exists to reconcile). ``body_changed`` covers the gmail-thread
|
|
580
|
+
# in-place upsert case where ``created`` stays ``False`` but the body
|
|
581
|
+
# was rewritten — without it the on-disk mirror would lag the DB.
|
|
582
|
+
if (
|
|
583
|
+
vault_root is not None
|
|
584
|
+
and result.document_id is not None
|
|
585
|
+
and (result.created or result.body_changed or force)
|
|
586
|
+
):
|
|
587
|
+
try:
|
|
588
|
+
# ``force=True`` because we just inserted/replaced the DB row —
|
|
589
|
+
# we know a change happened. Without it, the body-hash skip in
|
|
590
|
+
# ``_write_doc_file`` would mask any frontmatter-only delta
|
|
591
|
+
# (relevant when ``--force`` rewrites a doc with the same body
|
|
592
|
+
# but different metadata).
|
|
593
|
+
regenerate_vault_file(
|
|
594
|
+
conn, result.document_id, vault_path=vault_root, force=True
|
|
595
|
+
)
|
|
596
|
+
except OSError as exc:
|
|
597
|
+
# Only OSError is reachable here: ``regenerate_vault_file``'s
|
|
598
|
+
# ValueError paths (no document with id, kind='vault') are both
|
|
599
|
+
# impossible for a row we just inserted with default
|
|
600
|
+
# ``kind='ingested'``. Catching ValueError would mask unrelated
|
|
601
|
+
# bugs.
|
|
602
|
+
_logger.warning(
|
|
603
|
+
"vault mirror write failed for document %s: %s; "
|
|
604
|
+
"DB ingest succeeded — recover via `brain vault export`",
|
|
605
|
+
result.document_id,
|
|
606
|
+
exc,
|
|
607
|
+
)
|
|
608
|
+
|
|
609
|
+
# Wave G1-c — people-aspect graph sync. Runs OUTSIDE the DB transaction
|
|
610
|
+
# (post-commit, reusing ``conn``) on the same gate as the vault mirror:
|
|
611
|
+
# only when an actual write happened (created / body changed / forced).
|
|
612
|
+
# ``GraphSyncer.reconcile`` is best-effort and never raises — a graph-sync
|
|
613
|
+
# failure is logged and swallowed so a committed ingest is never undone
|
|
614
|
+
# (the graph is a recomputable mirror). A ``None`` syncer (library callers,
|
|
615
|
+
# tests) or a disabled / AGE-absent DB is a no-op.
|
|
616
|
+
if (
|
|
617
|
+
graph_syncer is not None
|
|
618
|
+
and result.document_id is not None
|
|
619
|
+
and (result.created or result.body_changed or force)
|
|
620
|
+
):
|
|
621
|
+
graph_syncer.reconcile(conn, result.document_id)
|
|
622
|
+
# Force content-hash-fallback replaced an old row under a new uuid — drop
|
|
623
|
+
# the old uuid's stale AGE Document vertex + any persons it orphaned. Runs
|
|
624
|
+
# AFTER reconcile(new) so shared persons keep their vertex; best-effort /
|
|
625
|
+
# never-raises like every other graph-sync call.
|
|
626
|
+
if graph_syncer is not None and result.replaced_document_id is not None:
|
|
627
|
+
graph_syncer.remove(conn, result.replaced_document_id)
|
|
628
|
+
|
|
629
|
+
return result
|
|
630
|
+
|
|
631
|
+
|
|
632
|
+
@dataclass(frozen=True)
|
|
633
|
+
class _IngestAction:
|
|
634
|
+
"""The write the ingest pipeline will perform, decided by a read-only pass.
|
|
635
|
+
|
|
636
|
+
Computed by :func:`_resolve_ingest_action` BEFORE embeddings are generated
|
|
637
|
+
so the slow Ollama embed call (and the write transaction) is skipped
|
|
638
|
+
entirely on the no-op ``skip`` path. ``existing_id`` is the target row for
|
|
639
|
+
``skip`` / ``update`` / ``replace`` and ``None`` for ``insert``.
|
|
640
|
+
"""
|
|
641
|
+
|
|
642
|
+
kind: str # "skip" | "update" | "insert" | "replace"
|
|
643
|
+
existing_id: str | None
|
|
644
|
+
body_changed: bool
|
|
645
|
+
|
|
646
|
+
|
|
647
|
+
def _resolve_ingest_action(
|
|
648
|
+
conn: psycopg.Connection,
|
|
649
|
+
*,
|
|
650
|
+
doc: ExtractedDoc,
|
|
651
|
+
source_kind: str,
|
|
652
|
+
source_external_id: str | None,
|
|
653
|
+
content_hash: str,
|
|
654
|
+
force: bool,
|
|
655
|
+
draft: bool,
|
|
656
|
+
is_thread: bool,
|
|
657
|
+
) -> _IngestAction:
|
|
658
|
+
"""Read-only dedup decision — runs OUTSIDE the write transaction.
|
|
659
|
+
|
|
660
|
+
Replicates the dedup ladder previously embedded inside
|
|
661
|
+
:func:`_ingest_within_transaction` (Gmail thread → file ``source_path`` →
|
|
662
|
+
sourced-stdin → content-hash fallback) but performs only SELECTs, so the
|
|
663
|
+
caller can short-circuit ``skip`` (no embed, no transaction, no enrich) and
|
|
664
|
+
defer the embed for ``update`` / ``insert`` / ``replace`` until after the
|
|
665
|
+
slow Ollama call has run unlocked (Wave perf-T3 / F1).
|
|
666
|
+
|
|
667
|
+
Decisions, mirroring the legacy branches exactly:
|
|
668
|
+
|
|
669
|
+
- Gmail thread: draft-only-on-published → ``skip`` (Q1-A partial-window
|
|
670
|
+
no-op); same-hash-no-force → ``skip``; else ``update``
|
|
671
|
+
(``body_changed = existing_hash != content_hash``); no row → ``insert``.
|
|
672
|
+
- File ``source_path``: same-hash-no-force → ``skip``; else ``update``; no
|
|
673
|
+
row → ``insert`` (never reaches the content-hash fallback — migration 006
|
|
674
|
+
scopes the content_hash UNIQUE index to ``source_path IS NULL``).
|
|
675
|
+
- Sourced-stdin: >1 rows → :class:`IngestAmbiguousSource`; same-hash-no-force
|
|
676
|
+
→ ``skip``; else ``update``; no row → content-hash fallback.
|
|
677
|
+
- Content-hash fallback (stdin only): row + no force → ``skip``; row + force
|
|
678
|
+
→ ``replace`` (DELETE old + INSERT new uuid); no row → ``insert``.
|
|
679
|
+
|
|
680
|
+
The sourced branch drops the legacy ``FOR UPDATE OF d`` lock — the
|
|
681
|
+
authoritative row lock is re-acquired inside :func:`_update_doc_in_place` in
|
|
682
|
+
the write transaction. As before, this pipeline is safe for sequential CLI
|
|
683
|
+
use only; a concurrent dual-INSERT of the same key still raises
|
|
684
|
+
IntegrityError on the partial unique indexes.
|
|
685
|
+
"""
|
|
686
|
+
h = content_hash
|
|
687
|
+
if is_thread:
|
|
688
|
+
thread_id = doc.metadata.get("thread_id")
|
|
689
|
+
assert isinstance(thread_id, str) and thread_id # _is_gmail_thread_doc
|
|
690
|
+
existing = conn.execute(
|
|
691
|
+
"SELECT id, content_hash, draft FROM documents "
|
|
692
|
+
"WHERE thread_id=%s AND kind='ingested' AND content_type='email_thread'",
|
|
693
|
+
(thread_id,),
|
|
694
|
+
).fetchone()
|
|
695
|
+
if existing is not None:
|
|
696
|
+
existing_id, existing_hash, existing_draft = existing
|
|
697
|
+
if draft and not existing_draft:
|
|
698
|
+
return _IngestAction("skip", str(existing_id), False)
|
|
699
|
+
if not force and existing_hash == h:
|
|
700
|
+
return _IngestAction("skip", str(existing_id), False)
|
|
701
|
+
return _IngestAction("update", str(existing_id), existing_hash != h)
|
|
702
|
+
return _IngestAction("insert", None, True)
|
|
703
|
+
|
|
704
|
+
if doc.source_path is not None:
|
|
705
|
+
existing = conn.execute(
|
|
706
|
+
"SELECT id, content_hash FROM documents "
|
|
707
|
+
"WHERE source_path=%s AND kind='ingested'",
|
|
708
|
+
(doc.source_path,),
|
|
709
|
+
).fetchone()
|
|
710
|
+
if existing:
|
|
711
|
+
existing_id, existing_hash = existing
|
|
712
|
+
if not force and existing_hash == h:
|
|
713
|
+
return _IngestAction("skip", str(existing_id), False)
|
|
714
|
+
return _IngestAction("update", str(existing_id), existing_hash != h)
|
|
715
|
+
return _IngestAction("insert", None, True)
|
|
716
|
+
|
|
717
|
+
# source_path is None — sourced lookup first, then content-hash fallback.
|
|
718
|
+
if source_external_id is not None:
|
|
719
|
+
sourced_rows = conn.execute(
|
|
720
|
+
"""
|
|
721
|
+
SELECT d.id, d.content_hash
|
|
722
|
+
FROM documents d
|
|
723
|
+
JOIN sources s ON d.source_id = s.id
|
|
724
|
+
WHERE s.kind = %s AND s.external_id = %s
|
|
725
|
+
""",
|
|
726
|
+
(source_kind, source_external_id),
|
|
727
|
+
).fetchall()
|
|
728
|
+
if len(sourced_rows) > 1:
|
|
729
|
+
raise IngestAmbiguousSource(
|
|
730
|
+
f"Multiple documents share source ({source_kind!r}, "
|
|
731
|
+
f"{source_external_id!r}): "
|
|
732
|
+
f"{[str(r[0]) for r in sourced_rows]}. "
|
|
733
|
+
"Resolve manually before re-ingesting."
|
|
734
|
+
)
|
|
735
|
+
if sourced_rows:
|
|
736
|
+
existing_id, existing_hash = sourced_rows[0]
|
|
737
|
+
if not force and existing_hash == h:
|
|
738
|
+
return _IngestAction("skip", str(existing_id), False)
|
|
739
|
+
return _IngestAction("update", str(existing_id), existing_hash != h)
|
|
740
|
+
|
|
741
|
+
existing = conn.execute(
|
|
742
|
+
"SELECT id FROM documents "
|
|
743
|
+
"WHERE content_hash=%s AND kind='ingested' AND source_path IS NULL",
|
|
744
|
+
(h,),
|
|
745
|
+
).fetchone()
|
|
746
|
+
if existing:
|
|
747
|
+
if not force:
|
|
748
|
+
return _IngestAction("skip", str(existing[0]), False)
|
|
749
|
+
return _IngestAction("replace", str(existing[0]), True)
|
|
750
|
+
return _IngestAction("insert", None, True)
|
|
751
|
+
|
|
752
|
+
|
|
753
|
+
def _insert_new_document(
|
|
754
|
+
conn: psycopg.Connection,
|
|
755
|
+
*,
|
|
756
|
+
doc: ExtractedDoc,
|
|
757
|
+
source_kind: str,
|
|
758
|
+
source_external_id: str | None,
|
|
759
|
+
source_metadata: dict[str, Any],
|
|
760
|
+
tags: list[str],
|
|
761
|
+
content_hash: str,
|
|
762
|
+
draft: bool,
|
|
763
|
+
chunks: list[Chunk],
|
|
764
|
+
embeddings: list[list[float] | None],
|
|
765
|
+
) -> str:
|
|
766
|
+
"""INSERT a brand-new document row + its chunks; return the new id.
|
|
767
|
+
|
|
768
|
+
Runs inside the caller's open transaction. Embeddings are PRECOMPUTED by the
|
|
769
|
+
caller (outside the transaction, Wave perf-T3 / F1), so this helper performs
|
|
770
|
+
no Ollama calls and the transaction holds no locks across network I/O.
|
|
771
|
+
Auto-summary enrichment AND the slow Krisp calendar/contacts refresh are NOT
|
|
772
|
+
run here — the caller invokes :func:`_enrich_post_ingest_hook` and
|
|
773
|
+
:func:`_run_post_commit_source_hooks` AFTER the commit (Task 2.11), so the
|
|
774
|
+
Krisp ``gws`` subprocess calls never hold the write transaction open. The
|
|
775
|
+
fast in-transaction Gmail directory upsert (:func:`_run_source_hooks`) DOES
|
|
776
|
+
run here, preserving its atomic-rollback contract.
|
|
777
|
+
"""
|
|
778
|
+
source_id = _upsert_source(
|
|
779
|
+
conn,
|
|
780
|
+
kind=source_kind,
|
|
781
|
+
external_id=source_external_id,
|
|
782
|
+
metadata=source_metadata,
|
|
783
|
+
)
|
|
784
|
+
|
|
785
|
+
# Pre-insert: derive source-specific metadata fields (e.g. Krisp
|
|
786
|
+
# ``_participant_keys``) so they're stored alongside the doc row in one
|
|
787
|
+
# INSERT. The leading underscore flags the key as derived/internal.
|
|
788
|
+
_apply_pre_insert_metadata(doc, source_kind=source_kind)
|
|
789
|
+
|
|
790
|
+
# Project email/krisp metadata onto typed columns. The base INSERT always
|
|
791
|
+
# writes the same fixed columns; the promoted columns ride along as a
|
|
792
|
+
# dynamic suffix so a doc with no recognized metadata keys produces the
|
|
793
|
+
# exact same SQL as before migration 007.
|
|
794
|
+
promoted = _promote_metadata_to_columns(doc.metadata)
|
|
795
|
+
promoted_columns = list(promoted.keys())
|
|
796
|
+
promoted_values = [promoted[c] for c in promoted_columns]
|
|
797
|
+
extra_cols = "".join(f", {c}" for c in promoted_columns)
|
|
798
|
+
extra_placeholders = ", %s" * len(promoted_columns)
|
|
799
|
+
|
|
800
|
+
doc_row = conn.execute(
|
|
801
|
+
f"""
|
|
802
|
+
INSERT INTO documents (source_id, title, content, content_hash, content_type,
|
|
803
|
+
source_path, tags, metadata, draft{extra_cols})
|
|
804
|
+
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s{extra_placeholders})
|
|
805
|
+
RETURNING id
|
|
806
|
+
""",
|
|
807
|
+
(
|
|
808
|
+
source_id,
|
|
809
|
+
doc.title,
|
|
810
|
+
doc.content,
|
|
811
|
+
content_hash,
|
|
812
|
+
doc.content_type,
|
|
813
|
+
doc.source_path,
|
|
814
|
+
tags,
|
|
815
|
+
json.dumps(doc.metadata),
|
|
816
|
+
draft,
|
|
817
|
+
*promoted_values,
|
|
818
|
+
),
|
|
819
|
+
).fetchone()
|
|
820
|
+
assert doc_row is not None # RETURNING id always yields a row
|
|
821
|
+
document_id = str(doc_row[0])
|
|
822
|
+
|
|
823
|
+
title_text, tags_text = _chunk_search_metadata(doc.title, tags)
|
|
824
|
+
_insert_chunks(
|
|
825
|
+
conn,
|
|
826
|
+
document_id=document_id,
|
|
827
|
+
chunks=chunks,
|
|
828
|
+
embeddings=embeddings,
|
|
829
|
+
title_text=title_text,
|
|
830
|
+
tags_text=tags_text,
|
|
831
|
+
)
|
|
832
|
+
|
|
833
|
+
# Wave Q1-D — Krisp action-items docs always carry the ``action-items`` tag
|
|
834
|
+
# so ``brain list --tag action-items`` and ``brain todo`` work out of the
|
|
835
|
+
# box. Mechanical, not LLM-derived; never re-proposed by ``brain tag
|
|
836
|
+
# --auto`` (D19 + D20).
|
|
837
|
+
_maybe_autotag_action_items(conn, document_id=document_id, doc=doc, tags=tags)
|
|
838
|
+
|
|
839
|
+
# In-transaction source hook (Gmail directory upsert). Its rollback contract
|
|
840
|
+
# is intentional — a directory-write failure rolls the document back. The
|
|
841
|
+
# slow Krisp calendar/contacts refresh runs POST-COMMIT instead — see
|
|
842
|
+
# :func:`_ingest_within_transaction` (Task 2.11).
|
|
843
|
+
_run_source_hooks(
|
|
844
|
+
conn,
|
|
845
|
+
source_kind=source_kind,
|
|
846
|
+
doc=doc,
|
|
847
|
+
document_id=document_id,
|
|
848
|
+
)
|
|
849
|
+
return document_id
|
|
850
|
+
|
|
851
|
+
|
|
852
|
+
def _ingest_within_transaction(
|
|
853
|
+
conn: psycopg.Connection,
|
|
854
|
+
*,
|
|
855
|
+
embedder: Embedder,
|
|
856
|
+
doc: ExtractedDoc,
|
|
857
|
+
source_kind: str,
|
|
858
|
+
source_external_id: str | None,
|
|
859
|
+
source_metadata: dict[str, Any],
|
|
860
|
+
tags: list[str],
|
|
861
|
+
force: bool,
|
|
862
|
+
gws_runner: GwsRunner | None,
|
|
863
|
+
content_hash: str,
|
|
864
|
+
draft: bool = False,
|
|
865
|
+
enricher: OllamaEnricher | None = None,
|
|
866
|
+
enrich: bool = True,
|
|
867
|
+
enrich_min_tokens: int = 50,
|
|
868
|
+
) -> IngestResult:
|
|
869
|
+
"""Decide the dedup action, embed OUTSIDE the transaction, then write.
|
|
870
|
+
|
|
871
|
+
Wave perf-T3 (F1) restructures the ingest hot path so neither slow Ollama
|
|
872
|
+
call holds a DB lock:
|
|
873
|
+
|
|
874
|
+
1. :func:`_resolve_ingest_action` runs the read-only dedup ladder. A
|
|
875
|
+
``skip`` decision (same content_hash / draft partial-window no-op)
|
|
876
|
+
returns immediately — no embed, no transaction, no enrich (matches the
|
|
877
|
+
prior early-return short-circuits exactly).
|
|
878
|
+
2. ``embedder.embed`` runs BEFORE ``with conn.transaction()`` opens, so the
|
|
879
|
+
2–5 s Ollama embed never holds a row lock. Skipped when there are no
|
|
880
|
+
chunks. (Previously the in-place UPDATE path embedded while holding a
|
|
881
|
+
``FOR UPDATE`` lock on the document row.)
|
|
882
|
+
3. The transaction performs ONLY the INSERT/UPDATE of the document + its
|
|
883
|
+
chunks using the precomputed vectors — it commits in ~milliseconds.
|
|
884
|
+
4. Auto-summary enrichment runs AFTER the commit, in autocommit mode, as
|
|
885
|
+
its own short ``UPDATE``. A post-commit enrichment failure leaves the
|
|
886
|
+
committed document with ``summary IS NULL`` (best-effort; recoverable via
|
|
887
|
+
``brain enrich --backfill``) — it can never roll back the ingest.
|
|
888
|
+
|
|
889
|
+
Atomicity preserved: the document row + ALL its chunk rows still INSERT (or
|
|
890
|
+
UPDATE + re-chunk) inside ONE transaction, so a write failure rolls back the
|
|
891
|
+
whole document. An embed failure raises BEFORE the transaction opens (no
|
|
892
|
+
partial row, no orphaned chunks). Sequential-CLI dedup guarantees are
|
|
893
|
+
unchanged; the partial-window draft no-op is enforced in BOTH the decision
|
|
894
|
+
pass and inside :func:`_update_doc_in_place` (defense in depth).
|
|
895
|
+
"""
|
|
896
|
+
h = content_hash
|
|
897
|
+
is_thread = _is_gmail_thread_doc(doc, source_kind)
|
|
898
|
+
|
|
899
|
+
action = _resolve_ingest_action(
|
|
900
|
+
conn,
|
|
901
|
+
doc=doc,
|
|
902
|
+
source_kind=source_kind,
|
|
903
|
+
source_external_id=source_external_id,
|
|
904
|
+
content_hash=h,
|
|
905
|
+
force=force,
|
|
906
|
+
draft=draft,
|
|
907
|
+
is_thread=is_thread,
|
|
908
|
+
)
|
|
909
|
+
if action.kind == "skip":
|
|
910
|
+
return IngestResult(
|
|
911
|
+
document_id=action.existing_id, created=False, body_changed=False
|
|
912
|
+
)
|
|
913
|
+
|
|
914
|
+
# Chunk (cheap, pure Python) then embed — BOTH outside the transaction so
|
|
915
|
+
# the Ollama HTTP round-trip holds no DB lock.
|
|
916
|
+
chunks = chunk_text(doc.content, count_tokens=embedder.count_tokens)
|
|
917
|
+
if action.kind in ("insert", "replace") and not chunks:
|
|
918
|
+
# Matches the legacy INSERT-path "no chunks → no document" behavior.
|
|
919
|
+
# (The update path still proceeds on empty chunks — it rebuilds zero
|
|
920
|
+
# chunk rows, preserving the prior in-place behavior.)
|
|
921
|
+
return IngestResult(document_id=None, created=False)
|
|
922
|
+
embeddings = _embed_chunks(embedder, [c.content for c in chunks])
|
|
923
|
+
|
|
924
|
+
replaced_id: str | None = None
|
|
925
|
+
did_write = True
|
|
926
|
+
with conn.transaction():
|
|
927
|
+
if action.kind == "update":
|
|
928
|
+
assert action.existing_id is not None
|
|
929
|
+
did_write = _update_doc_in_place(
|
|
930
|
+
conn,
|
|
931
|
+
document_id=action.existing_id,
|
|
932
|
+
doc=doc,
|
|
933
|
+
source_kind=source_kind,
|
|
934
|
+
source_external_id=source_external_id,
|
|
935
|
+
source_metadata=source_metadata,
|
|
936
|
+
tags=tags,
|
|
937
|
+
content_hash=h,
|
|
938
|
+
body_changed=action.body_changed,
|
|
939
|
+
draft=draft,
|
|
940
|
+
chunks=chunks,
|
|
941
|
+
embeddings=embeddings,
|
|
942
|
+
)
|
|
943
|
+
document_id = action.existing_id
|
|
944
|
+
created = False
|
|
945
|
+
# When the partial-window draft guard no-op'd, nothing was written.
|
|
946
|
+
body_changed = action.body_changed if did_write else False
|
|
947
|
+
else: # "insert" or "replace"
|
|
948
|
+
if action.kind == "replace":
|
|
949
|
+
assert action.existing_id is not None
|
|
950
|
+
# Force re-ingest: replace the old row under a new uuid. Capture
|
|
951
|
+
# the old id so the post-commit graph hook can DETACH DELETE its
|
|
952
|
+
# now-orphaned AGE vertex (relational graph rows cascade with the
|
|
953
|
+
# row; AGE doesn't).
|
|
954
|
+
replaced_id = action.existing_id
|
|
955
|
+
conn.execute(
|
|
956
|
+
"DELETE FROM documents WHERE id=%s", (action.existing_id,)
|
|
957
|
+
)
|
|
958
|
+
document_id = _insert_new_document(
|
|
959
|
+
conn,
|
|
960
|
+
doc=doc,
|
|
961
|
+
source_kind=source_kind,
|
|
962
|
+
source_external_id=source_external_id,
|
|
963
|
+
source_metadata=source_metadata,
|
|
964
|
+
tags=tags,
|
|
965
|
+
content_hash=h,
|
|
966
|
+
draft=draft,
|
|
967
|
+
chunks=chunks,
|
|
968
|
+
embeddings=embeddings,
|
|
969
|
+
)
|
|
970
|
+
created = True
|
|
971
|
+
body_changed = True
|
|
972
|
+
|
|
973
|
+
# --- Post-commit Krisp source hook (Task 2.11). ---
|
|
974
|
+
# The Krisp calendar+contacts refresh runs AFTER the document transaction
|
|
975
|
+
# commits, on the same ``did_write`` gate the enrich step uses. Previously it
|
|
976
|
+
# ran INSIDE ``conn.transaction()``, so the hook held the write transaction
|
|
977
|
+
# open across multi-second ``gws`` subprocess calls (calendar + contacts
|
|
978
|
+
# refresh). Moving it post-commit loses no rollback semantics: both refresh
|
|
979
|
+
# helpers soft-degrade (catch runner failures and return 0). The Gmail
|
|
980
|
+
# directory upsert deliberately stays IN-TRANSACTION (fast local write with
|
|
981
|
+
# an atomic-rollback contract) — it already ran inside the transaction above.
|
|
982
|
+
# ``did_write`` is False only on the partial-window draft no-op, which wrote
|
|
983
|
+
# nothing — matching the prior in-transaction gating.
|
|
984
|
+
if did_write:
|
|
985
|
+
_run_post_commit_source_hooks(
|
|
986
|
+
conn,
|
|
987
|
+
source_kind=source_kind,
|
|
988
|
+
doc=doc,
|
|
989
|
+
document_id=document_id,
|
|
990
|
+
gws_runner=gws_runner,
|
|
991
|
+
)
|
|
992
|
+
|
|
993
|
+
# --- Post-commit auto-summary (Wave perf-T3 / F1). ---
|
|
994
|
+
# Runs in autocommit (the transaction has committed) as its own short
|
|
995
|
+
# UPDATE. ``is_new_doc=created`` skips the redundant idempotency SELECT on
|
|
996
|
+
# the create path (F4) — a freshly-INSERTed row is known to have
|
|
997
|
+
# ``summary IS NULL``. The update path keeps the SELECT for D11 idempotency.
|
|
998
|
+
# ``did_write`` is False only when :func:`_update_doc_in_place` hit its
|
|
999
|
+
# partial-window draft guard, in which case no body was written and we must
|
|
1000
|
+
# not enrich. Skip rules + failure handling (Ollama down → WARN, ingest
|
|
1001
|
+
# already committed) live inside the hook.
|
|
1002
|
+
if did_write:
|
|
1003
|
+
_enrich_post_ingest_hook(
|
|
1004
|
+
conn,
|
|
1005
|
+
document_id=document_id,
|
|
1006
|
+
doc=doc,
|
|
1007
|
+
enricher=enricher,
|
|
1008
|
+
enrich=enrich,
|
|
1009
|
+
min_tokens=enrich_min_tokens,
|
|
1010
|
+
content_hash=h,
|
|
1011
|
+
body_changed=body_changed,
|
|
1012
|
+
is_new_doc=created,
|
|
1013
|
+
)
|
|
1014
|
+
|
|
1015
|
+
return IngestResult(
|
|
1016
|
+
document_id=document_id,
|
|
1017
|
+
created=created,
|
|
1018
|
+
body_changed=body_changed,
|
|
1019
|
+
replaced_document_id=replaced_id,
|
|
1020
|
+
)
|
|
1021
|
+
|
|
1022
|
+
|
|
1023
|
+
def _update_doc_in_place(
|
|
1024
|
+
conn: psycopg.Connection,
|
|
1025
|
+
*,
|
|
1026
|
+
document_id: str,
|
|
1027
|
+
doc: ExtractedDoc,
|
|
1028
|
+
source_kind: str,
|
|
1029
|
+
source_external_id: str | None,
|
|
1030
|
+
source_metadata: dict[str, Any],
|
|
1031
|
+
tags: list[str],
|
|
1032
|
+
content_hash: str,
|
|
1033
|
+
body_changed: bool,
|
|
1034
|
+
chunks: list[Chunk],
|
|
1035
|
+
embeddings: list[list[float] | None],
|
|
1036
|
+
draft: bool = False,
|
|
1037
|
+
) -> bool:
|
|
1038
|
+
"""Replace title / body / metadata / tags / typed columns on an existing
|
|
1039
|
+
document row and rebuild its chunks. Returns ``True`` when the row was
|
|
1040
|
+
written, ``False`` when the partial-window draft guard no-op'd.
|
|
1041
|
+
|
|
1042
|
+
Used by the gmail-thread upsert path (P2.2), the file-path UPDATE-in-place
|
|
1043
|
+
branch, and the sourced-stdin (Krisp/Slack) UPDATE-in-place branch. Runs
|
|
1044
|
+
inside the caller's open transaction. The document UUID is preserved so any
|
|
1045
|
+
``links`` / ``derived_links`` / ``unresolved_links`` referencing this doc
|
|
1046
|
+
keep pointing at the same row across re-ingest. Old chunks are deleted via
|
|
1047
|
+
direct DELETE (no need for ON DELETE CASCADE — we know the parent doc id)
|
|
1048
|
+
and re-inserted from the precomputed vectors. The Gmail directory upsert
|
|
1049
|
+
re-runs IN-TRANSACTION so headers from the latest message in the rebuilt
|
|
1050
|
+
thread propagate (its atomic-rollback contract is preserved); the slow Krisp
|
|
1051
|
+
calendar/contacts refresh runs POST-COMMIT — see
|
|
1052
|
+
:func:`_ingest_within_transaction` (Task 2.11).
|
|
1053
|
+
|
|
1054
|
+
Wave perf-T3 (F1): chunks + embeddings are PRECOMPUTED by the caller
|
|
1055
|
+
(outside the transaction) and passed in — this helper performs no Ollama
|
|
1056
|
+
calls, so the ``FOR UPDATE`` row lock it holds is never held across network
|
|
1057
|
+
I/O. Auto-summary enrichment is NOT run here; the caller invokes
|
|
1058
|
+
:func:`_enrich_post_ingest_hook` AFTER the commit.
|
|
1059
|
+
|
|
1060
|
+
Tag union semantics: existing curated tags on the row are preserved and
|
|
1061
|
+
merged with the incoming ``tags`` argument via
|
|
1062
|
+
``normalize_tags([*existing, *incoming])``. Incoming tags are added;
|
|
1063
|
+
existing tags are never removed — ``brain tag`` is the only explicit
|
|
1064
|
+
tag-modification surface.
|
|
1065
|
+
|
|
1066
|
+
Summary-staleness rule: when ``body_changed`` is True this NULLs out
|
|
1067
|
+
``summary`` / ``summary_model`` / ``summary_at`` in the same UPDATE that
|
|
1068
|
+
overwrites ``content_hash``, so a stale summary describing the old body can
|
|
1069
|
+
never linger if the post-commit enrich hook can't regenerate (``enrich=False``
|
|
1070
|
+
/ ``enricher=None`` / content < min_tokens). When it can, the hook overwrites
|
|
1071
|
+
it. The caller passes ``body_changed`` on to the post-commit hook so its D11
|
|
1072
|
+
idempotency check knows the prior summary is stale. (Codex finding 1,
|
|
1073
|
+
2026-05-11; Codex stop-gate finding, 2026-05-13.)
|
|
1074
|
+
"""
|
|
1075
|
+
# F5 (Wave perf-T3): one locked SELECT reads BOTH the draft flag (for the
|
|
1076
|
+
# partial-window guard) AND the existing tags (for the union merge), where
|
|
1077
|
+
# the prior code issued two separate SELECTs against the same row. The
|
|
1078
|
+
# ``FOR UPDATE`` lock guards against concurrent tag-writes. ``body_changed``
|
|
1079
|
+
# is signalled via the kwarg, so we don't read content_hash here — the
|
|
1080
|
+
# caller computed it from the pre-UPDATE hash diff.
|
|
1081
|
+
existing_row = conn.execute(
|
|
1082
|
+
"SELECT draft, tags FROM documents WHERE id=%s FOR UPDATE",
|
|
1083
|
+
(document_id,),
|
|
1084
|
+
).fetchone()
|
|
1085
|
+
|
|
1086
|
+
# Partial-window guard: if the incoming extraction is all-draft but the
|
|
1087
|
+
# existing row is published, the ingest window is a subset of the full
|
|
1088
|
+
# thread (e.g. ``brain ingest-gmail --since 7d`` where only a fresh
|
|
1089
|
+
# draft reply falls in the window but the thread's sent messages are
|
|
1090
|
+
# older). The draft-only extraction is NOT authoritative — refusing the
|
|
1091
|
+
# UPDATE preserves the published body, metadata, chunks, and links.
|
|
1092
|
+
# The legitimate auto-flip direction (TRUE→FALSE when a sent reply
|
|
1093
|
+
# arrives) is unaffected because this guard only triggers on draft=True.
|
|
1094
|
+
if draft and existing_row is not None and not existing_row[0]:
|
|
1095
|
+
return False
|
|
1096
|
+
|
|
1097
|
+
existing_tags: list[str] = (
|
|
1098
|
+
list(existing_row[1]) if existing_row and existing_row[1] else []
|
|
1099
|
+
)
|
|
1100
|
+
|
|
1101
|
+
# Union semantics: preserve existing curated tags; add incoming tags.
|
|
1102
|
+
# normalize_tags is idempotent (casefold + hyphenate) and preserves
|
|
1103
|
+
# first-seen insertion order, matching how apply_tags composes tags.
|
|
1104
|
+
merged_tags = normalize_tags([*existing_tags, *tags])
|
|
1105
|
+
|
|
1106
|
+
# Re-evaluate the source row — same ``(kind, external_id)`` as the
|
|
1107
|
+
# initial insert, so this is an ON CONFLICT no-op in practice. Kept here
|
|
1108
|
+
# so a future caller passing different ``source_metadata`` still resolves
|
|
1109
|
+
# the source row.
|
|
1110
|
+
source_id = _upsert_source(
|
|
1111
|
+
conn,
|
|
1112
|
+
kind=source_kind,
|
|
1113
|
+
external_id=source_external_id,
|
|
1114
|
+
metadata=source_metadata,
|
|
1115
|
+
)
|
|
1116
|
+
|
|
1117
|
+
# Pre-insert metadata derivation (krisp adds ``_participant_keys``).
|
|
1118
|
+
# No-op for gmail today, but kept symmetric with the INSERT path so
|
|
1119
|
+
# future per-source enrichments apply on UPDATE too.
|
|
1120
|
+
_apply_pre_insert_metadata(doc, source_kind=source_kind)
|
|
1121
|
+
|
|
1122
|
+
promoted = _promote_metadata_to_columns(doc.metadata)
|
|
1123
|
+
set_parts: list[str] = [
|
|
1124
|
+
"source_id=%s",
|
|
1125
|
+
"title=%s",
|
|
1126
|
+
"content=%s",
|
|
1127
|
+
"content_hash=%s",
|
|
1128
|
+
"content_type=%s",
|
|
1129
|
+
"tags=%s",
|
|
1130
|
+
"metadata=%s::jsonb",
|
|
1131
|
+
# Bump ingested_at on every content update so `brain status`'s
|
|
1132
|
+
# `last ingest` and the homepage recent-rail reflect actual ingest
|
|
1133
|
+
# activity, not just the original creation time. Default-NOW() on
|
|
1134
|
+
# INSERT means newly-created rows still get the right value.
|
|
1135
|
+
"ingested_at=NOW()",
|
|
1136
|
+
]
|
|
1137
|
+
params: list[Any] = [
|
|
1138
|
+
source_id,
|
|
1139
|
+
doc.title,
|
|
1140
|
+
doc.content,
|
|
1141
|
+
content_hash,
|
|
1142
|
+
doc.content_type,
|
|
1143
|
+
merged_tags,
|
|
1144
|
+
json.dumps(doc.metadata),
|
|
1145
|
+
]
|
|
1146
|
+
|
|
1147
|
+
# When the body changed, NULL out the stored summary so a stale summary
|
|
1148
|
+
# describing the OLD body can never be left on disk. The post-ingest
|
|
1149
|
+
# enrich hook regenerates the summary from NULL when an enricher is
|
|
1150
|
+
# available; when it isn't (enrich=False, enricher=None, or content
|
|
1151
|
+
# shorter than min_tokens) the hook returns early — but the row is
|
|
1152
|
+
# now correctly NULL-summary rather than carrying a misleading
|
|
1153
|
+
# description of the previous body. Complements master's body_changed
|
|
1154
|
+
# kwarg fix to _enrich_post_ingest_hook, which handles the
|
|
1155
|
+
# "regenerate when possible" path; this handles the "can't regenerate,
|
|
1156
|
+
# don't lie" path. (Codex stop-gate finding, 2026-05-13.)
|
|
1157
|
+
if body_changed:
|
|
1158
|
+
set_parts.extend(["summary=NULL", "summary_model=NULL", "summary_at=NULL"])
|
|
1159
|
+
|
|
1160
|
+
# Write the draft column directly. The entry-level partial-window
|
|
1161
|
+
# guard above already returned early if (existing=published,
|
|
1162
|
+
# incoming=draft-only), so reaching this point means either:
|
|
1163
|
+
# (a) draft=False — a normal sent-message update (auto-flip TRUE→FALSE
|
|
1164
|
+
# included), or
|
|
1165
|
+
# (b) draft=True AND existing was also draft=True (legitimate
|
|
1166
|
+
# all-draft refresh, e.g. a new draft was added to the thread).
|
|
1167
|
+
# In both cases the incoming ``draft`` value is authoritative.
|
|
1168
|
+
set_parts.append("draft=%s")
|
|
1169
|
+
params.append(draft)
|
|
1170
|
+
# Project the rebuilt thread's metadata onto the typed columns so they
|
|
1171
|
+
# stay in lockstep with the JSONB blob: a key that's present in the
|
|
1172
|
+
# new metadata writes the column; a key that's gone (e.g. an old
|
|
1173
|
+
# ``in_reply_to`` that no longer applies after a thread split) clears
|
|
1174
|
+
# the column to NULL.
|
|
1175
|
+
for column in _PROMOTED_COLUMNS:
|
|
1176
|
+
if column in promoted:
|
|
1177
|
+
set_parts.append(f"{column}=%s")
|
|
1178
|
+
params.append(promoted[column])
|
|
1179
|
+
else:
|
|
1180
|
+
set_parts.append(f"{column}=NULL")
|
|
1181
|
+
params.append(document_id)
|
|
1182
|
+
conn.execute(
|
|
1183
|
+
f"UPDATE documents SET {', '.join(set_parts)} WHERE id=%s",
|
|
1184
|
+
params,
|
|
1185
|
+
)
|
|
1186
|
+
|
|
1187
|
+
conn.execute("DELETE FROM chunks WHERE document_id=%s", (document_id,))
|
|
1188
|
+
# Use merged_tags (not the incoming tags arg) so chunks.tags_text reflects
|
|
1189
|
+
# the full union — existing curated tags stay searchable via the weighted-B
|
|
1190
|
+
# tsv field even after a re-ingest that passes an empty tags list.
|
|
1191
|
+
title_text, tags_text = _chunk_search_metadata(doc.title, merged_tags)
|
|
1192
|
+
_insert_chunks(
|
|
1193
|
+
conn,
|
|
1194
|
+
document_id=document_id,
|
|
1195
|
+
chunks=chunks,
|
|
1196
|
+
embeddings=embeddings,
|
|
1197
|
+
title_text=title_text,
|
|
1198
|
+
tags_text=tags_text,
|
|
1199
|
+
)
|
|
1200
|
+
|
|
1201
|
+
# In-transaction source hook: the Gmail directory upsert re-runs so headers
|
|
1202
|
+
# from the latest message in the rebuilt thread propagate; its rollback
|
|
1203
|
+
# contract is intentional. The slow Krisp calendar/contacts refresh runs
|
|
1204
|
+
# POST-COMMIT — see :func:`_ingest_within_transaction` (Task 2.11).
|
|
1205
|
+
_run_source_hooks(
|
|
1206
|
+
conn,
|
|
1207
|
+
source_kind=source_kind,
|
|
1208
|
+
doc=doc,
|
|
1209
|
+
document_id=document_id,
|
|
1210
|
+
)
|
|
1211
|
+
return True
|
|
1212
|
+
|
|
1213
|
+
|
|
1214
|
+
def _maybe_autotag_action_items(
|
|
1215
|
+
conn: psycopg.Connection,
|
|
1216
|
+
*,
|
|
1217
|
+
document_id: str,
|
|
1218
|
+
doc: ExtractedDoc,
|
|
1219
|
+
tags: list[str],
|
|
1220
|
+
) -> None:
|
|
1221
|
+
"""Apply the ``action-items`` tag to ``krisp_action_items`` docs.
|
|
1222
|
+
|
|
1223
|
+
Mechanical (no LLM), idempotent (the tag rewrite below collapses
|
|
1224
|
+
duplicates), and silent on non-action-items docs. Called once after
|
|
1225
|
+
the documents INSERT in :func:`_ingest_within_transaction`.
|
|
1226
|
+
"""
|
|
1227
|
+
if doc.content_type != "krisp_action_items":
|
|
1228
|
+
return
|
|
1229
|
+
if "action-items" in tags:
|
|
1230
|
+
return # already present in the caller-supplied list
|
|
1231
|
+
# apply_tags re-normalizes; doing this in-transaction keeps the
|
|
1232
|
+
# documents row + tags atomic with the original ingest.
|
|
1233
|
+
apply_tags(conn, document_id, add=["action-items"])
|
|
1234
|
+
|
|
1235
|
+
|
|
1236
|
+
def _enrich_post_ingest_hook(
|
|
1237
|
+
conn: psycopg.Connection,
|
|
1238
|
+
*,
|
|
1239
|
+
document_id: str,
|
|
1240
|
+
doc: ExtractedDoc,
|
|
1241
|
+
enricher: OllamaEnricher | None,
|
|
1242
|
+
enrich: bool,
|
|
1243
|
+
min_tokens: int,
|
|
1244
|
+
content_hash: str,
|
|
1245
|
+
body_changed: bool = False,
|
|
1246
|
+
is_new_doc: bool = False,
|
|
1247
|
+
) -> None:
|
|
1248
|
+
"""Generate + persist a 2-3 sentence summary on ``documents.summary``.
|
|
1249
|
+
|
|
1250
|
+
On the ingest hot path (Wave perf-T3 / F1) this runs AFTER the document
|
|
1251
|
+
transaction has committed, in autocommit mode, as its own short ``UPDATE``
|
|
1252
|
+
— a failure here leaves the committed row with ``summary IS NULL`` and never
|
|
1253
|
+
rolls back the ingest. On the ``update_document`` (``brain edit``) path and
|
|
1254
|
+
in direct unit tests it still runs inside the caller's open transaction.
|
|
1255
|
+
Skips silently (with a debug log) when any of the wave-plan §3.a rules
|
|
1256
|
+
apply:
|
|
1257
|
+
|
|
1258
|
+
1. ``enrich=False`` — caller passed ``--no-enrich`` or a library caller
|
|
1259
|
+
deliberately disabled it.
|
|
1260
|
+
2. ``enricher is None`` — caller didn't wire one up (legit for tests
|
|
1261
|
+
and internal pipelines that don't want LLM round-trips).
|
|
1262
|
+
3. Content shorter than ``min_tokens`` tokens — the title alone is
|
|
1263
|
+
already a fine summary; no point spending an LLM round-trip.
|
|
1264
|
+
4. ``documents.summary IS NOT NULL`` AND ``content_hash`` unchanged
|
|
1265
|
+
AND ``body_changed`` is False — idempotency: same body, same model
|
|
1266
|
+
→ reuse the prior summary (D11).
|
|
1267
|
+
5. ``OllamaUnavailable`` — Ollama is down / 5xx; logged at WARN. The
|
|
1268
|
+
ingest still commits; ``brain enrich --backfill`` picks the row up
|
|
1269
|
+
later.
|
|
1270
|
+
6. :class:`EnrichmentError` (model returned malformed JSON twice in a
|
|
1271
|
+
row) — logged at WARN; row stays unenriched.
|
|
1272
|
+
|
|
1273
|
+
The ``body_changed`` kwarg is the Codex finding 1 (2026-05-11) fix.
|
|
1274
|
+
Update-path callers (:func:`_update_thread_doc_in_place`,
|
|
1275
|
+
:func:`update_document`) UPDATE ``documents.content_hash`` to the new
|
|
1276
|
+
value BEFORE invoking this hook. The hook then SELECTs the row, reads
|
|
1277
|
+
the just-overwritten hash, and the ``existing_hash == content_hash``
|
|
1278
|
+
comparison is meaningless — both sides equal the new hash. The
|
|
1279
|
+
pre-fix behavior fell through the D11 skip and left the prior
|
|
1280
|
+
(stale) summary in place even when the body had changed, which
|
|
1281
|
+
Q2-SUMMARY-WIKI then rendered above the article body as a wiki
|
|
1282
|
+
lede — user-visible drift. The fix: callers that know the body
|
|
1283
|
+
changed (or might have) pass ``body_changed=True``; the D11 skip
|
|
1284
|
+
only fires when ``body_changed`` is False AND the other conditions
|
|
1285
|
+
hold. The INSERT call site keeps the default ``False`` because the
|
|
1286
|
+
row is freshly INSERTed with ``summary IS NULL`` — the first clause
|
|
1287
|
+
of the D11 guard naturally short-circuits there.
|
|
1288
|
+
"""
|
|
1289
|
+
if not enrich:
|
|
1290
|
+
return
|
|
1291
|
+
if enricher is None:
|
|
1292
|
+
_logger.debug(
|
|
1293
|
+
"enrich hook: no enricher supplied for %s; skipping", document_id
|
|
1294
|
+
)
|
|
1295
|
+
return
|
|
1296
|
+
if enricher.count_tokens(doc.content) < min_tokens:
|
|
1297
|
+
return
|
|
1298
|
+
# F4 (Wave perf-T3): on the new-doc path the row was just INSERTed with
|
|
1299
|
+
# ``summary IS NULL`` / ``content_hash == content_hash`` / ``summary_model
|
|
1300
|
+
# IS NULL``, so the D11 idempotency SELECT can only ever read those known
|
|
1301
|
+
# values and never skips — drop the round-trip and proceed straight to
|
|
1302
|
+
# summarization. The update path (``is_new_doc=False``) keeps the SELECT so
|
|
1303
|
+
# D11 can reuse a still-valid summary.
|
|
1304
|
+
if not is_new_doc:
|
|
1305
|
+
row = conn.execute(
|
|
1306
|
+
"SELECT summary, content_hash, summary_model FROM documents WHERE id=%s",
|
|
1307
|
+
(document_id,),
|
|
1308
|
+
).fetchone()
|
|
1309
|
+
if row is None:
|
|
1310
|
+
return # defensive — the row was just INSERTed
|
|
1311
|
+
existing_summary, existing_hash, existing_model = row
|
|
1312
|
+
# Idempotency D11: reuse prior summary ONLY when the body AND model both
|
|
1313
|
+
# match the current enricher AND the caller hasn't told us the body
|
|
1314
|
+
# just changed. After a ``BRAIN_ENRICH_MODEL`` upgrade (e.g.,
|
|
1315
|
+
# llama3.1:8b → llama3.2:8b), the existing summary is stale relative
|
|
1316
|
+
# to the new model — re-enrich so improvements propagate.
|
|
1317
|
+
if (
|
|
1318
|
+
existing_summary is not None
|
|
1319
|
+
and not body_changed
|
|
1320
|
+
and existing_hash == content_hash
|
|
1321
|
+
and existing_model == enricher.model
|
|
1322
|
+
):
|
|
1323
|
+
return
|
|
1324
|
+
try:
|
|
1325
|
+
result = enricher.summarize(doc.title, doc.content)
|
|
1326
|
+
except OllamaUnavailable as exc:
|
|
1327
|
+
_logger.warning(
|
|
1328
|
+
"auto-summary skipped for %s: Ollama unavailable (%s); "
|
|
1329
|
+
"run `brain enrich --backfill` later",
|
|
1330
|
+
document_id,
|
|
1331
|
+
exc,
|
|
1332
|
+
)
|
|
1333
|
+
return
|
|
1334
|
+
except EnrichmentError as exc:
|
|
1335
|
+
_logger.warning(
|
|
1336
|
+
"auto-summary failed for %s: %s; row marked unenriched",
|
|
1337
|
+
document_id,
|
|
1338
|
+
exc,
|
|
1339
|
+
)
|
|
1340
|
+
return
|
|
1341
|
+
conn.execute(
|
|
1342
|
+
"UPDATE documents SET summary=%s, summary_model=%s, summary_at=NOW() "
|
|
1343
|
+
"WHERE id=%s",
|
|
1344
|
+
(result.summary, result.model, document_id),
|
|
1345
|
+
)
|
|
1346
|
+
|
|
1347
|
+
|
|
1348
|
+
def _apply_pre_insert_metadata(doc: ExtractedDoc, *, source_kind: str) -> None:
|
|
1349
|
+
"""Mutate ``doc.metadata`` in place to add derived fields by source.
|
|
1350
|
+
|
|
1351
|
+
Currently:
|
|
1352
|
+
- ``krisp`` → ``_participant_keys`` (sorted list of normalized speaker
|
|
1353
|
+
keys parsed from the transcript body). Always present after this call,
|
|
1354
|
+
even if the body has no speaker labels (empty list signals "this doc
|
|
1355
|
+
was processed by the linker pre-stage" to downstream code).
|
|
1356
|
+
"""
|
|
1357
|
+
if source_kind == "krisp":
|
|
1358
|
+
doc.metadata["_participant_keys"] = sorted(extract_krisp_speakers(doc.content))
|
|
1359
|
+
|
|
1360
|
+
|
|
1361
|
+
def _gmail_post_ingest_hook(
|
|
1362
|
+
conn: psycopg.Connection, doc: ExtractedDoc, document_id: str
|
|
1363
|
+
) -> None:
|
|
1364
|
+
"""Upsert (display_name, email) pairs from the Gmail From/To headers.
|
|
1365
|
+
|
|
1366
|
+
Runs inside the outer ``conn.transaction()`` opened by
|
|
1367
|
+
:func:`ingest_document` so a directory-write failure rolls the
|
|
1368
|
+
document back too. ``document_id`` is unused today but kept in the
|
|
1369
|
+
signature for symmetry with future hooks that may need it.
|
|
1370
|
+
"""
|
|
1371
|
+
del document_id # symmetry with other source hooks
|
|
1372
|
+
store = DirectoryStore(conn)
|
|
1373
|
+
for display_name, email in extract_gmail_addresses(doc.metadata):
|
|
1374
|
+
store.upsert_pair(
|
|
1375
|
+
display_name=display_name,
|
|
1376
|
+
email=email,
|
|
1377
|
+
source="gmail",
|
|
1378
|
+
)
|
|
1379
|
+
|
|
1380
|
+
|
|
1381
|
+
def _krisp_post_ingest_hook(
|
|
1382
|
+
conn: psycopg.Connection,
|
|
1383
|
+
doc: ExtractedDoc,
|
|
1384
|
+
document_id: str,
|
|
1385
|
+
runner: GwsRunner | None,
|
|
1386
|
+
) -> None:
|
|
1387
|
+
"""Trigger an incremental Calendar refresh + a stale-only Contacts refresh.
|
|
1388
|
+
|
|
1389
|
+
Both refreshes degrade soft: ``refresh_calendar`` / ``refresh_contacts``
|
|
1390
|
+
catch runner failures internally and return 0 so a Krisp ingest never
|
|
1391
|
+
fails on a transient gws hiccup.
|
|
1392
|
+
|
|
1393
|
+
Calendar window:
|
|
1394
|
+
- First run (no ``directory_refresh_state`` row for ``calendar``) →
|
|
1395
|
+
since = ``YYYY-01-01T00:00:00+00:00`` (current-year start, UTC).
|
|
1396
|
+
- Subsequent runs → since = the stored ``last_refreshed_at``.
|
|
1397
|
+
- until = ``datetime.now(tz=UTC)`` (always).
|
|
1398
|
+
|
|
1399
|
+
Contacts cadence:
|
|
1400
|
+
- Runs only if no state row exists OR ``last_refreshed_at`` is older
|
|
1401
|
+
than 24 hours. Prevents re-fetching the People API on every ingest.
|
|
1402
|
+
"""
|
|
1403
|
+
del doc, document_id # unused — krisp metadata mutation happens pre-insert
|
|
1404
|
+
if runner is None:
|
|
1405
|
+
_logger.warning(
|
|
1406
|
+
"krisp post-ingest: no gws_runner provided; "
|
|
1407
|
+
"skipping calendar/contacts refresh"
|
|
1408
|
+
)
|
|
1409
|
+
return
|
|
1410
|
+
|
|
1411
|
+
now = datetime.now(tz=UTC)
|
|
1412
|
+
|
|
1413
|
+
cal_row = conn.execute(
|
|
1414
|
+
"SELECT last_refreshed_at FROM directory_refresh_state "
|
|
1415
|
+
"WHERE source = 'calendar'"
|
|
1416
|
+
).fetchone()
|
|
1417
|
+
if cal_row is not None and cal_row[0] is not None:
|
|
1418
|
+
since = cal_row[0]
|
|
1419
|
+
else:
|
|
1420
|
+
since = datetime(now.year, 1, 1, tzinfo=UTC)
|
|
1421
|
+
refresh_calendar(conn, since=since, until=now, runner=runner)
|
|
1422
|
+
|
|
1423
|
+
contacts_row = conn.execute(
|
|
1424
|
+
"SELECT last_refreshed_at FROM directory_refresh_state "
|
|
1425
|
+
"WHERE source = 'contacts'"
|
|
1426
|
+
).fetchone()
|
|
1427
|
+
contacts_stale = (
|
|
1428
|
+
contacts_row is None
|
|
1429
|
+
or contacts_row[0] is None
|
|
1430
|
+
or contacts_row[0] < now - _CONTACTS_REFRESH_INTERVAL
|
|
1431
|
+
)
|
|
1432
|
+
if contacts_stale:
|
|
1433
|
+
refresh_contacts(conn, runner=runner)
|
|
1434
|
+
|
|
1435
|
+
|
|
1436
|
+
def _run_source_hooks(
|
|
1437
|
+
conn: psycopg.Connection,
|
|
1438
|
+
*,
|
|
1439
|
+
source_kind: str,
|
|
1440
|
+
doc: ExtractedDoc,
|
|
1441
|
+
document_id: str,
|
|
1442
|
+
) -> None:
|
|
1443
|
+
"""Dispatch the IN-TRANSACTION source-specific post-ingest hooks.
|
|
1444
|
+
|
|
1445
|
+
Only Gmail's directory upsert runs here — it is a fast local DB write
|
|
1446
|
+
(no network / subprocess) with a deliberate atomic-rollback contract: a
|
|
1447
|
+
directory-write failure rolls the whole document back
|
|
1448
|
+
(``test_directory_upsert_runs_in_same_transaction``). Keeping it in the
|
|
1449
|
+
write transaction is a preserved guarantee.
|
|
1450
|
+
|
|
1451
|
+
Krisp's calendar / contacts refresh does NOT run here — it shells out to
|
|
1452
|
+
``gws`` (multi-second subprocess / network calls) and would otherwise hold
|
|
1453
|
+
the write transaction open, so it runs POST-COMMIT via
|
|
1454
|
+
:func:`_run_post_commit_source_hooks` (Task 2.11).
|
|
1455
|
+
|
|
1456
|
+
Sources without an in-transaction hook are a no-op. A new fast/atomic hook
|
|
1457
|
+
adds a branch here; a new slow / soft-degrading hook adds a branch to
|
|
1458
|
+
:func:`_run_post_commit_source_hooks` instead (Open/Closed).
|
|
1459
|
+
"""
|
|
1460
|
+
if source_kind == "gmail":
|
|
1461
|
+
_gmail_post_ingest_hook(conn, doc, document_id)
|
|
1462
|
+
|
|
1463
|
+
|
|
1464
|
+
def _run_post_commit_source_hooks(
|
|
1465
|
+
conn: psycopg.Connection,
|
|
1466
|
+
*,
|
|
1467
|
+
source_kind: str,
|
|
1468
|
+
doc: ExtractedDoc,
|
|
1469
|
+
document_id: str,
|
|
1470
|
+
gws_runner: GwsRunner | None,
|
|
1471
|
+
) -> None:
|
|
1472
|
+
"""Dispatch the POST-COMMIT source-specific post-ingest hooks (Task 2.11).
|
|
1473
|
+
|
|
1474
|
+
Only Krisp's calendar + contacts refresh runs here. It shells out to the
|
|
1475
|
+
``gws`` CLI (multi-second subprocess / network calls) and soft-degrades —
|
|
1476
|
+
``refresh_calendar`` / ``refresh_contacts`` catch runner failures and
|
|
1477
|
+
return 0 — so running it AFTER the document transaction commits holds no
|
|
1478
|
+
write lock across the slow refresh and loses no rollback semantics.
|
|
1479
|
+
|
|
1480
|
+
Gmail's directory upsert runs IN-TRANSACTION via :func:`_run_source_hooks`
|
|
1481
|
+
because it has an atomic-rollback contract; keeping it there is a
|
|
1482
|
+
deliberately preserved guarantee, NOT an oversight.
|
|
1483
|
+
|
|
1484
|
+
**Post-commit failure containment.** This runs AFTER the document
|
|
1485
|
+
transaction has already committed (Task 2.11), so a failure here must never
|
|
1486
|
+
surface as an ingest failure — the row is durably saved. The refresh helpers
|
|
1487
|
+
already soft-degrade on ``gws``/runner failures, but the hook still issues its
|
|
1488
|
+
own ``directory_refresh_state`` SELECTs, whose raw ``psycopg.Error`` (e.g. a
|
|
1489
|
+
connection dropped after commit) would otherwise propagate out of
|
|
1490
|
+
:func:`ingest_document`. We catch it, log a WARN, and return — mirroring the
|
|
1491
|
+
:func:`_enrich_post_ingest_hook` guard (the calendar/contacts refresh is
|
|
1492
|
+
simply skipped; a later ingest re-triggers it).
|
|
1493
|
+
"""
|
|
1494
|
+
try:
|
|
1495
|
+
if source_kind == "krisp":
|
|
1496
|
+
_krisp_post_ingest_hook(conn, doc, document_id, gws_runner)
|
|
1497
|
+
except psycopg.Error as exc:
|
|
1498
|
+
_logger.warning(
|
|
1499
|
+
"post-commit source hook for %s failed (%s); ingest already "
|
|
1500
|
+
"committed — calendar/contacts refresh skipped",
|
|
1501
|
+
source_kind,
|
|
1502
|
+
exc,
|
|
1503
|
+
)
|
|
1504
|
+
|
|
1505
|
+
|
|
1506
|
+
def apply_tags(
|
|
1507
|
+
conn: psycopg.Connection,
|
|
1508
|
+
document_id: str,
|
|
1509
|
+
*,
|
|
1510
|
+
add: list[str] | None = None,
|
|
1511
|
+
remove: list[str] | None = None,
|
|
1512
|
+
) -> list[str]:
|
|
1513
|
+
"""Add and/or remove tags on a document; return the resulting tag list.
|
|
1514
|
+
|
|
1515
|
+
``add`` is unioned with the existing tags (idempotent — re-adding an
|
|
1516
|
+
existing tag is a no-op); ``remove`` strips any matching tags. Operations
|
|
1517
|
+
run in a single transaction. Caller is responsible for resolving any
|
|
1518
|
+
UUID prefix to a full ``document_id`` before calling.
|
|
1519
|
+
|
|
1520
|
+
Inputs are passed through :func:`brain.tags.normalize_tags` before the
|
|
1521
|
+
DB write — this is the single ongoing-enforcement boundary that keeps
|
|
1522
|
+
``brain tag <id> +COMPANY_REDACTED`` storing ``company-ko`` and lets a remove of
|
|
1523
|
+
``COMPANY_REDACTED`` match a row that's currently stored as ``company-ko``.
|
|
1524
|
+
"""
|
|
1525
|
+
# Deferred import: ``brain.queries`` imports the ``Embedder`` Protocol
|
|
1526
|
+
# from this module, so a top-level ``from ..queries import ...`` would
|
|
1527
|
+
# cycle at package-load time. The function-local import resolves cleanly
|
|
1528
|
+
# because both modules have finished initializing by the time
|
|
1529
|
+
# ``apply_tags`` is first called.
|
|
1530
|
+
from ..queries import sync_chunk_search_metadata
|
|
1531
|
+
|
|
1532
|
+
add = normalize_tags(add or [])
|
|
1533
|
+
remove = normalize_tags(remove or [])
|
|
1534
|
+
with conn.transaction():
|
|
1535
|
+
if add:
|
|
1536
|
+
conn.execute(
|
|
1537
|
+
"UPDATE documents SET tags = ARRAY(SELECT DISTINCT unnest(tags || %s::text[])) "
|
|
1538
|
+
"WHERE id = %s",
|
|
1539
|
+
(add, document_id),
|
|
1540
|
+
)
|
|
1541
|
+
if remove:
|
|
1542
|
+
conn.execute(
|
|
1543
|
+
"UPDATE documents SET tags = ARRAY(SELECT t FROM unnest(tags) AS t "
|
|
1544
|
+
"WHERE t <> ALL(%s::text[])) WHERE id = %s",
|
|
1545
|
+
(remove, document_id),
|
|
1546
|
+
)
|
|
1547
|
+
# Migration 009 denormalizes documents.tags onto chunks.tags_text so
|
|
1548
|
+
# the weighted tsv reflects tag changes. Run inside the same
|
|
1549
|
+
# transaction that wrote the new tags so a rollback (e.g. SELECT
|
|
1550
|
+
# below failing) keeps chunks and parent in lockstep. The IS
|
|
1551
|
+
# DISTINCT FROM guards in the helper make this a no-op when add /
|
|
1552
|
+
# remove produced no net change.
|
|
1553
|
+
if add or remove:
|
|
1554
|
+
sync_chunk_search_metadata(conn, document_id)
|
|
1555
|
+
row = conn.execute(
|
|
1556
|
+
"SELECT tags FROM documents WHERE id=%s", (document_id,)
|
|
1557
|
+
).fetchone()
|
|
1558
|
+
if row is None:
|
|
1559
|
+
raise ValueError(f"document not found: {document_id}")
|
|
1560
|
+
return list(row[0] or [])
|
|
1561
|
+
|
|
1562
|
+
|
|
1563
|
+
_MIRROR_FRONTMATTER_FIELDS = frozenset(
|
|
1564
|
+
{"title", "tags", "metadata", "content_type", "draft", "summary"}
|
|
1565
|
+
)
|
|
1566
|
+
|
|
1567
|
+
|
|
1568
|
+
def update_document(
|
|
1569
|
+
conn: psycopg.Connection,
|
|
1570
|
+
*,
|
|
1571
|
+
document_id: str,
|
|
1572
|
+
embedder: Embedder | None = None,
|
|
1573
|
+
new_title: str | None = None,
|
|
1574
|
+
new_content_type: str | None = None,
|
|
1575
|
+
new_content: str | None = None,
|
|
1576
|
+
metadata_patch: dict[str, Any] | None = None,
|
|
1577
|
+
replace_metadata: bool = False,
|
|
1578
|
+
new_tags: list[str] | None = None,
|
|
1579
|
+
new_draft: bool | None = None,
|
|
1580
|
+
vault_root: Path | None = None,
|
|
1581
|
+
enricher: OllamaEnricher | None = None,
|
|
1582
|
+
enrich: bool = True,
|
|
1583
|
+
enrich_min_tokens: int = 50,
|
|
1584
|
+
graph_syncer: GraphSyncer | None = None,
|
|
1585
|
+
) -> UpdateResult:
|
|
1586
|
+
"""Update one document in place.
|
|
1587
|
+
|
|
1588
|
+
Body changes (``new_content``) re-chunk + re-embed atomically: the prior
|
|
1589
|
+
chunks are deleted and new ones are inserted in the same transaction.
|
|
1590
|
+
Metadata defaults to a shallow merge — top-level keys overwrite, nested
|
|
1591
|
+
objects are not deep-merged. Set ``replace_metadata=True`` to swap the
|
|
1592
|
+
blob entirely.
|
|
1593
|
+
|
|
1594
|
+
``new_draft`` flips the top-level ``documents.draft`` boolean column
|
|
1595
|
+
introduced by migration 007. Unlike metadata-promoted columns this lives
|
|
1596
|
+
directly on ``documents``; passing ``True`` / ``False`` writes the new
|
|
1597
|
+
value, ``None`` leaves it alone. The wiki build (Quartz contentIndex
|
|
1598
|
+
emitter) hides ``draft=true`` docs from the explorer/graph/search;
|
|
1599
|
+
``brain search`` / ``brain list`` still surface them so the user can
|
|
1600
|
+
re-publish.
|
|
1601
|
+
|
|
1602
|
+
Raises :class:`ValueError` if ``new_content`` is empty/whitespace-only or
|
|
1603
|
+
if its SHA-256 collides with another document. ``embedder`` is required
|
|
1604
|
+
when ``new_content`` is provided. Empty/no-op edits are not an error and
|
|
1605
|
+
return an :class:`UpdateResult` with ``fields_changed=[]``.
|
|
1606
|
+
|
|
1607
|
+
Vault mirror: when ``vault_root`` is supplied AND the edit actually
|
|
1608
|
+
changed the body or any frontmatter-bearing field
|
|
1609
|
+
(``title`` / ``tags`` / ``metadata`` / ``content_type`` / ``draft``),
|
|
1610
|
+
the mirror file under ``vault_root`` is regenerated via
|
|
1611
|
+
:func:`brain.vault.export.regenerate_vault_file`. Vault-tier rows
|
|
1612
|
+
(``kind='vault'``) are skipped via a DB pre-check — those files are
|
|
1613
|
+
file-source-of-truth and ``vault sync`` reconciles back to the DB.
|
|
1614
|
+
A filesystem error (``OSError``) on the mirror write logs a WARNING
|
|
1615
|
+
and does NOT roll back the DB update — drift is recoverable via
|
|
1616
|
+
``brain vault export --force``. Named ``vault_root`` to distinguish
|
|
1617
|
+
from the per-document ``documents.vault_path`` relative path.
|
|
1618
|
+
|
|
1619
|
+
Wave G1-c graph sync: when ``graph_syncer`` is supplied AND any field
|
|
1620
|
+
changed, the document's people-aspect graph is reconciled post-commit via
|
|
1621
|
+
:meth:`brain.graph_rag.sync.GraphSyncer.reconcile` (reusing ``conn``,
|
|
1622
|
+
OUTSIDE the transaction). Best-effort / never-raises; a ``None`` syncer or a
|
|
1623
|
+
no-op edit skips it.
|
|
1624
|
+
"""
|
|
1625
|
+
with conn.transaction():
|
|
1626
|
+
row = conn.execute(
|
|
1627
|
+
"SELECT title, content, content_type, metadata, tags, kind, draft "
|
|
1628
|
+
"FROM documents WHERE id=%s",
|
|
1629
|
+
(document_id,),
|
|
1630
|
+
).fetchone()
|
|
1631
|
+
if row is None:
|
|
1632
|
+
raise ValueError(f"document not found: {document_id}")
|
|
1633
|
+
(
|
|
1634
|
+
cur_title,
|
|
1635
|
+
cur_content,
|
|
1636
|
+
cur_type,
|
|
1637
|
+
cur_meta,
|
|
1638
|
+
cur_tags,
|
|
1639
|
+
cur_kind,
|
|
1640
|
+
cur_draft,
|
|
1641
|
+
) = row
|
|
1642
|
+
cur_meta = dict(cur_meta or {})
|
|
1643
|
+
cur_tags = list(cur_tags or [])
|
|
1644
|
+
|
|
1645
|
+
fields_changed: list[str] = []
|
|
1646
|
+
sets: list[str] = []
|
|
1647
|
+
params: list[Any] = []
|
|
1648
|
+
|
|
1649
|
+
rechunked = False
|
|
1650
|
+
new_hash: str | None = None
|
|
1651
|
+
if new_content is not None:
|
|
1652
|
+
if embedder is None:
|
|
1653
|
+
raise ValueError("embedder is required when new_content is provided")
|
|
1654
|
+
stripped = new_content.strip()
|
|
1655
|
+
if not stripped:
|
|
1656
|
+
raise ValueError("content is empty")
|
|
1657
|
+
if new_content != cur_content:
|
|
1658
|
+
new_hash = _content_hash(new_content)
|
|
1659
|
+
clash = conn.execute(
|
|
1660
|
+
"SELECT id FROM documents WHERE content_hash=%s AND id<>%s",
|
|
1661
|
+
(new_hash, document_id),
|
|
1662
|
+
).fetchone()
|
|
1663
|
+
if clash:
|
|
1664
|
+
raise ValueError(
|
|
1665
|
+
f"content collides with existing document {clash[0]}"
|
|
1666
|
+
)
|
|
1667
|
+
rechunked = True
|
|
1668
|
+
|
|
1669
|
+
if new_title is not None and new_title != cur_title:
|
|
1670
|
+
sets.append("title=%s")
|
|
1671
|
+
params.append(new_title)
|
|
1672
|
+
fields_changed.append("title")
|
|
1673
|
+
|
|
1674
|
+
if new_content_type is not None and new_content_type != cur_type:
|
|
1675
|
+
sets.append("content_type=%s")
|
|
1676
|
+
params.append(new_content_type)
|
|
1677
|
+
fields_changed.append("content_type")
|
|
1678
|
+
|
|
1679
|
+
if metadata_patch is not None:
|
|
1680
|
+
new_meta = (
|
|
1681
|
+
metadata_patch if replace_metadata else {**cur_meta, **metadata_patch}
|
|
1682
|
+
)
|
|
1683
|
+
if new_meta != cur_meta:
|
|
1684
|
+
sets.append("metadata=%s::jsonb")
|
|
1685
|
+
params.append(json.dumps(new_meta))
|
|
1686
|
+
fields_changed.append("metadata")
|
|
1687
|
+
# Re-project the new metadata blob onto typed columns, and
|
|
1688
|
+
# diff against what we'd project from the prior blob. This
|
|
1689
|
+
# keeps the typed columns in lockstep with metadata: a
|
|
1690
|
+
# patch that adds ``thread_id`` populates the column; a
|
|
1691
|
+
# replace_metadata that drops the key NULLs the column.
|
|
1692
|
+
old_promoted = _promote_metadata_to_columns(cur_meta)
|
|
1693
|
+
new_promoted = _promote_metadata_to_columns(new_meta)
|
|
1694
|
+
for column in _PROMOTED_COLUMNS:
|
|
1695
|
+
if column in new_promoted:
|
|
1696
|
+
if new_promoted[column] != old_promoted.get(column):
|
|
1697
|
+
sets.append(f"{column}=%s")
|
|
1698
|
+
params.append(new_promoted[column])
|
|
1699
|
+
elif column in old_promoted:
|
|
1700
|
+
# replace_metadata dropped the source key — clear
|
|
1701
|
+
# the typed column so it no longer disagrees with
|
|
1702
|
+
# the JSONB blob.
|
|
1703
|
+
sets.append(f"{column}=NULL")
|
|
1704
|
+
|
|
1705
|
+
if new_tags is not None and sorted(new_tags) != sorted(cur_tags):
|
|
1706
|
+
sets.append("tags=%s")
|
|
1707
|
+
params.append(list(new_tags))
|
|
1708
|
+
fields_changed.append("tags")
|
|
1709
|
+
|
|
1710
|
+
if new_draft is not None and bool(new_draft) != bool(cur_draft):
|
|
1711
|
+
sets.append("draft=%s")
|
|
1712
|
+
params.append(bool(new_draft))
|
|
1713
|
+
fields_changed.append("draft")
|
|
1714
|
+
|
|
1715
|
+
if rechunked:
|
|
1716
|
+
assert new_hash is not None # set above when rechunked is True
|
|
1717
|
+
assert embedder is not None # checked above
|
|
1718
|
+
sets.append("content=%s")
|
|
1719
|
+
params.append(new_content)
|
|
1720
|
+
sets.append("content_hash=%s")
|
|
1721
|
+
params.append(new_hash)
|
|
1722
|
+
fields_changed.append("content")
|
|
1723
|
+
# Same rationale as ``_update_doc_in_place``: NULL the stored
|
|
1724
|
+
# summary in the SAME UPDATE that overwrites content_hash so a
|
|
1725
|
+
# stale summary describing the old body can never linger when
|
|
1726
|
+
# the post-update enrich hook can't regenerate (``enrich=False``,
|
|
1727
|
+
# ``enricher=None``, or content < min_tokens). The hook below
|
|
1728
|
+
# regenerates from NULL when an enricher is available; otherwise
|
|
1729
|
+
# the row is correctly NULL-summary rather than a misleading
|
|
1730
|
+
# description of the previous content. (Codex stop-gate finding,
|
|
1731
|
+
# 2026-05-13.)
|
|
1732
|
+
sets.append("summary=NULL")
|
|
1733
|
+
sets.append("summary_model=NULL")
|
|
1734
|
+
sets.append("summary_at=NULL")
|
|
1735
|
+
|
|
1736
|
+
conn.execute(
|
|
1737
|
+
"DELETE FROM chunks WHERE document_id=%s", (document_id,)
|
|
1738
|
+
)
|
|
1739
|
+
assert new_content is not None # gated by the empty-check above
|
|
1740
|
+
chunks = chunk_text(new_content, count_tokens=embedder.count_tokens)
|
|
1741
|
+
if chunks:
|
|
1742
|
+
embeddings = _embed_chunks(embedder, [c.content for c in chunks])
|
|
1743
|
+
# Project the post-update title/tags onto the new chunks so
|
|
1744
|
+
# the weighted tsv reflects the user's new edits, not the
|
|
1745
|
+
# pre-edit state. Falls back to current values when the
|
|
1746
|
+
# caller didn't pass an override; the surrounding ``UPDATE
|
|
1747
|
+
# documents`` writes the same values back to the documents
|
|
1748
|
+
# row, so the chunks and parent stay in lockstep.
|
|
1749
|
+
final_title = new_title if new_title is not None else cur_title
|
|
1750
|
+
final_tags = list(new_tags) if new_tags is not None else cur_tags
|
|
1751
|
+
title_text, tags_text = _chunk_search_metadata(final_title, final_tags)
|
|
1752
|
+
# F2 (Wave perf-T3): single executemany over the rebuilt chunks
|
|
1753
|
+
# instead of a per-chunk round-trip. Still inside this txn — the
|
|
1754
|
+
# ``brain edit`` path is interactive/single-doc, not the batch
|
|
1755
|
+
# ingest hot path, so its embed/enrich stay transactional.
|
|
1756
|
+
_insert_chunks(
|
|
1757
|
+
conn,
|
|
1758
|
+
document_id=document_id,
|
|
1759
|
+
chunks=chunks,
|
|
1760
|
+
embeddings=embeddings,
|
|
1761
|
+
title_text=title_text,
|
|
1762
|
+
tags_text=tags_text,
|
|
1763
|
+
)
|
|
1764
|
+
|
|
1765
|
+
if sets:
|
|
1766
|
+
# Same rationale as update_document: bump ingested_at so
|
|
1767
|
+
# `brain status`'s last-ingest stat and the vault-export
|
|
1768
|
+
# `updated:` frontmatter field reflect actual edits.
|
|
1769
|
+
sets.append("ingested_at=NOW()")
|
|
1770
|
+
params.append(document_id)
|
|
1771
|
+
conn.execute(
|
|
1772
|
+
f"UPDATE documents SET {', '.join(sets)} WHERE id=%s",
|
|
1773
|
+
params,
|
|
1774
|
+
)
|
|
1775
|
+
|
|
1776
|
+
# Migration 009 denormalizes documents.title / documents.tags onto
|
|
1777
|
+
# chunks.title_text / chunks.tags_text. After ``rechunked`` the
|
|
1778
|
+
# freshly-inserted chunks already carry the post-update values, so
|
|
1779
|
+
# the helper's IS DISTINCT FROM guards turn this into a no-op for
|
|
1780
|
+
# those rows. For the no-rechunk path (title-only or tags-only
|
|
1781
|
+
# edit), this is the call that propagates the change to every
|
|
1782
|
+
# existing chunk so the weighted tsv stays consistent.
|
|
1783
|
+
if "title" in fields_changed or "tags" in fields_changed:
|
|
1784
|
+
from ..queries import sync_chunk_search_metadata
|
|
1785
|
+
sync_chunk_search_metadata(conn, document_id)
|
|
1786
|
+
|
|
1787
|
+
# Wave Q1-D — re-enrich on body change. The ``rechunked`` gate
|
|
1788
|
+
# above only fires when ``new_content != cur_content``, so reaching
|
|
1789
|
+
# this point means the body provably changed — pass
|
|
1790
|
+
# ``body_changed=True`` so the D11 idempotency check inside
|
|
1791
|
+
# ``_enrich_post_ingest_hook`` knows the existing summary is now
|
|
1792
|
+
# stale relative to the just-written body (Codex finding 1 fix —
|
|
1793
|
+
# without this kwarg the hook reads back the just-updated row
|
|
1794
|
+
# whose ``content_hash`` already matches ``new_hash``, the
|
|
1795
|
+
# ``existing_hash == content_hash`` check trivially passes, and
|
|
1796
|
+
# the hook short-circuits leaving the prior body's summary
|
|
1797
|
+
# rendered in the Q2-SUMMARY-WIKI lede above the new body).
|
|
1798
|
+
# Build an ExtractedDoc-shaped object lazily — we only need
|
|
1799
|
+
# ``title`` and ``content`` (the parts the enricher reads).
|
|
1800
|
+
# Re-uses the post-update title/content so an in-flight title
|
|
1801
|
+
# edit sees the new one.
|
|
1802
|
+
if rechunked:
|
|
1803
|
+
final_title = new_title if new_title is not None else cur_title
|
|
1804
|
+
assert new_content is not None # rechunked implies new_content set
|
|
1805
|
+
new_doc = ExtractedDoc(
|
|
1806
|
+
title=final_title,
|
|
1807
|
+
content=new_content,
|
|
1808
|
+
content_type=cur_type,
|
|
1809
|
+
source_path=None,
|
|
1810
|
+
metadata={},
|
|
1811
|
+
)
|
|
1812
|
+
assert new_hash is not None # rechunked implies new_hash set
|
|
1813
|
+
_enrich_post_ingest_hook(
|
|
1814
|
+
conn,
|
|
1815
|
+
document_id=document_id,
|
|
1816
|
+
doc=new_doc,
|
|
1817
|
+
enricher=enricher,
|
|
1818
|
+
enrich=enrich,
|
|
1819
|
+
min_tokens=enrich_min_tokens,
|
|
1820
|
+
content_hash=new_hash,
|
|
1821
|
+
body_changed=True,
|
|
1822
|
+
)
|
|
1823
|
+
|
|
1824
|
+
# Mirror writes happen OUTSIDE the transaction so a filesystem error
|
|
1825
|
+
# cannot roll back the DB update. Triggered when the body was rechunked
|
|
1826
|
+
# OR any frontmatter-derived field changed (the mirror's frontmatter is
|
|
1827
|
+
# built from documents.title/tags/metadata/content_type — a metadata-only
|
|
1828
|
+
# edit must still propagate). Vault-tier rows are skipped via the DB
|
|
1829
|
+
# ``cur_kind`` pre-check rather than by catching ``regenerate_vault_file``'s
|
|
1830
|
+
# ValueError — pre-checking ``kind`` from the DB is more robust than
|
|
1831
|
+
# catching a ``ValueError`` and string-matching its message, since the
|
|
1832
|
+
# upstream message can be rephrased without notice.
|
|
1833
|
+
needs_mirror = rechunked or any(
|
|
1834
|
+
f in _MIRROR_FRONTMATTER_FIELDS for f in fields_changed
|
|
1835
|
+
)
|
|
1836
|
+
if vault_root is not None and cur_kind != "vault" and needs_mirror:
|
|
1837
|
+
try:
|
|
1838
|
+
# ``force=True`` because we've already gated on
|
|
1839
|
+
# ``rechunked or any(... in _MIRROR_FRONTMATTER_FIELDS ...)`` —
|
|
1840
|
+
# a frontmatter-only edit (tags / metadata / content_type) leaves
|
|
1841
|
+
# the body unchanged, so the body-hash skip in ``_write_doc_file``
|
|
1842
|
+
# would silently drop the rewrite and leave stale frontmatter on
|
|
1843
|
+
# disk. The full-corpus ``export_vault`` keeps the default
|
|
1844
|
+
# ``force=False`` so re-runs of that path stay cheap.
|
|
1845
|
+
regenerate_vault_file(
|
|
1846
|
+
conn, document_id, vault_path=vault_root, force=True
|
|
1847
|
+
)
|
|
1848
|
+
except OSError as exc:
|
|
1849
|
+
# Only OSError is reachable: ``no document with id`` is impossible
|
|
1850
|
+
# (we just SELECTed it), and ``kind='vault'`` is gated above.
|
|
1851
|
+
_logger.warning(
|
|
1852
|
+
"vault mirror write failed for document %s: %s; "
|
|
1853
|
+
"DB update succeeded — recover via `brain vault export`",
|
|
1854
|
+
document_id,
|
|
1855
|
+
exc,
|
|
1856
|
+
)
|
|
1857
|
+
|
|
1858
|
+
# Wave G1-c — people-aspect graph sync after an in-place edit. Runs
|
|
1859
|
+
# post-commit (reusing ``conn``) whenever any field actually changed; a
|
|
1860
|
+
# change to the participants metadata flips the reconcile watermark and
|
|
1861
|
+
# re-indexes, while a person-irrelevant edit (e.g. a ``draft`` toggle) is a
|
|
1862
|
+
# cheap watermark-skip. Best-effort / never-raises — see
|
|
1863
|
+
# :meth:`brain.graph_rag.sync.GraphSyncer.reconcile`. A ``None`` syncer or a
|
|
1864
|
+
# no-op edit (empty ``fields_changed``) skips graph sync entirely.
|
|
1865
|
+
if graph_syncer is not None and fields_changed:
|
|
1866
|
+
graph_syncer.reconcile(conn, document_id)
|
|
1867
|
+
|
|
1868
|
+
return UpdateResult(
|
|
1869
|
+
document_id=document_id,
|
|
1870
|
+
fields_changed=fields_changed,
|
|
1871
|
+
rechunked=rechunked,
|
|
1872
|
+
)
|
|
1873
|
+
|
|
1874
|
+
|
|
1875
|
+
_EXTRACTORS = {
|
|
1876
|
+
".txt": "text",
|
|
1877
|
+
".md": "markdown",
|
|
1878
|
+
".markdown": "markdown",
|
|
1879
|
+
".pdf": "pdf",
|
|
1880
|
+
".docx": "docx",
|
|
1881
|
+
}
|
|
1882
|
+
|
|
1883
|
+
|
|
1884
|
+
def extract_path(path: Path) -> ExtractedDoc:
|
|
1885
|
+
"""Dispatch to the correct extractor based on file extension.
|
|
1886
|
+
|
|
1887
|
+
Raises ``ValueError`` for unsupported extensions or malformed input files;
|
|
1888
|
+
``OSError`` for unreadable files. Backend-specific parser errors (e.g.
|
|
1889
|
+
:class:`pypdf.errors.PyPdfError`) are wrapped as ``ValueError`` so callers
|
|
1890
|
+
only need to handle a narrow set of exception types.
|
|
1891
|
+
"""
|
|
1892
|
+
ext = Path(path).suffix.lower()
|
|
1893
|
+
name = _EXTRACTORS.get(ext)
|
|
1894
|
+
if name is None:
|
|
1895
|
+
raise ValueError(f"unsupported file type: {ext}")
|
|
1896
|
+
try:
|
|
1897
|
+
if name == "text":
|
|
1898
|
+
from .text import extract_text
|
|
1899
|
+
return extract_text(path)
|
|
1900
|
+
if name == "markdown":
|
|
1901
|
+
from .markdown import extract_markdown
|
|
1902
|
+
return extract_markdown(path)
|
|
1903
|
+
if name == "pdf":
|
|
1904
|
+
from pypdf.errors import PyPdfError
|
|
1905
|
+
|
|
1906
|
+
from .pdf import extract_pdf
|
|
1907
|
+
try:
|
|
1908
|
+
return extract_pdf(path)
|
|
1909
|
+
except PyPdfError as e:
|
|
1910
|
+
raise ValueError(f"malformed PDF: {e}") from e
|
|
1911
|
+
if name == "docx":
|
|
1912
|
+
from docx.opc.exceptions import PackageNotFoundError
|
|
1913
|
+
|
|
1914
|
+
from .docx import extract_docx
|
|
1915
|
+
try:
|
|
1916
|
+
return extract_docx(path)
|
|
1917
|
+
except PackageNotFoundError as e:
|
|
1918
|
+
raise ValueError(f"malformed DOCX: {e}") from e
|
|
1919
|
+
except UnicodeDecodeError as e: # pragma: no cover - extractors use errors="replace"
|
|
1920
|
+
raise ValueError(f"could not decode file as UTF-8: {e}") from e
|
|
1921
|
+
raise AssertionError("unreachable") # pragma: no cover
|
|
1922
|
+
|
|
1923
|
+
|
|
1924
|
+
def supported_extensions() -> list[str]:
|
|
1925
|
+
"""Return the list of file extensions that :func:`extract_path` can handle."""
|
|
1926
|
+
return list(_EXTRACTORS.keys())
|