secondbrain-py 0.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- brain/__init__.py +0 -0
- brain/__main__.py +18 -0
- brain/_capture_command.py +445 -0
- brain/_compose.py +52 -0
- brain/activity.py +206 -0
- brain/ask.py +631 -0
- brain/audio.py +591 -0
- brain/backfill/__init__.py +12 -0
- brain/backfill/search_extras.py +141 -0
- brain/backfill/source_rows.py +101 -0
- brain/bin/__init__.py +1 -0
- brain/bin/_launcher.py +107 -0
- brain/bin/down.py +8 -0
- brain/bin/launchd.py +268 -0
- brain/bin/monitor.py +570 -0
- brain/bin/rebuild.py +8 -0
- brain/bin/status.py +8 -0
- brain/bin/up.py +8 -0
- brain/brief.py +272 -0
- brain/capture.py +49 -0
- brain/chat.py +293 -0
- brain/cli.py +9760 -0
- brain/cli_claude.py +81 -0
- brain/cli_connect.py +285 -0
- brain/cli_demo.py +266 -0
- brain/config.py +1949 -0
- brain/connect.py +925 -0
- brain/db.py +540 -0
- brain/demo/__init__.py +452 -0
- brain/demo/corpus/manifest.json +403 -0
- brain/demo/embedder.py +74 -0
- brain/durations.py +84 -0
- brain/edit_session.py +156 -0
- brain/editor.py +67 -0
- brain/elicit/__init__.py +16 -0
- brain/elicit/detectors.py +250 -0
- brain/elicit/drafter.py +70 -0
- brain/elicit/queue.py +220 -0
- brain/elicit/schema.py +48 -0
- brain/elicit/session.py +445 -0
- brain/embedding_targets.py +54 -0
- brain/embeddings.py +424 -0
- brain/enrichment.py +808 -0
- brain/errors.py +357 -0
- brain/eval/__init__.py +129 -0
- brain/eval/answer_eval.py +281 -0
- brain/eval/baseline.py +265 -0
- brain/eval/concept_extraction.py +378 -0
- brain/eval/corpus.py +152 -0
- brain/eval/errors.py +19 -0
- brain/eval/graph_baseline.py +226 -0
- brain/eval/graph_retrieval.py +202 -0
- brain/eval/graph_runner.py +319 -0
- brain/eval/metrics.py +101 -0
- brain/eval/runner.py +223 -0
- brain/format.py +783 -0
- brain/gaps.py +390 -0
- brain/graph_rag/__init__.py +94 -0
- brain/graph_rag/_retrieval_common.py +113 -0
- brain/graph_rag/aggregates.py +303 -0
- brain/graph_rag/aliases/__init__.py +583 -0
- brain/graph_rag/backends/__init__.py +10 -0
- brain/graph_rag/backends/_age_helpers.py +473 -0
- brain/graph_rag/backends/age.py +782 -0
- brain/graph_rag/backends/base.py +272 -0
- brain/graph_rag/build.py +344 -0
- brain/graph_rag/communities.py +644 -0
- brain/graph_rag/communities_summary.py +437 -0
- brain/graph_rag/concepts.py +202 -0
- brain/graph_rag/cooccur.py +193 -0
- brain/graph_rag/cross_type.py +312 -0
- brain/graph_rag/extract.py +885 -0
- brain/graph_rag/fuse.py +371 -0
- brain/graph_rag/global_.py +412 -0
- brain/graph_rag/grouping.py +372 -0
- brain/graph_rag/person_resolver.py +167 -0
- brain/graph_rag/reconcile.py +792 -0
- brain/graph_rag/relational.py +353 -0
- brain/graph_rag/retrieve.py +526 -0
- brain/graph_rag/router.py +288 -0
- brain/graph_rag/schema.py +320 -0
- brain/graph_rag/sync.py +237 -0
- brain/graph_rag/tenancy.py +43 -0
- brain/graph_rag/themes.py +501 -0
- brain/graph_rag/weighting.py +202 -0
- brain/ingest/__init__.py +1926 -0
- brain/ingest/chunker.py +249 -0
- brain/ingest/docx.py +40 -0
- brain/ingest/gmail.py +621 -0
- brain/ingest/markdown.py +37 -0
- brain/ingest/pdf.py +61 -0
- brain/ingest/stdin.py +22 -0
- brain/ingest/sub_tokens.py +91 -0
- brain/ingest/text.py +16 -0
- brain/interactions.py +205 -0
- brain/maintenance.py +355 -0
- brain/mcp_server.py +3405 -0
- brain/migrations/001_init.sql +43 -0
- brain/migrations/002_qwen3_embedding.sql +17 -0
- brain/migrations/003_vault_model.sql +41 -0
- brain/migrations/004_relax_content_hash_uniqueness.sql +18 -0
- brain/migrations/005_derived_links.sql +67 -0
- brain/migrations/006_dedup_file_by_source_path.sql +25 -0
- brain/migrations/007_email_thread_and_draft.sql +15 -0
- brain/migrations/008_gmail_thread_unique.sql +11 -0
- brain/migrations/009_chunks_weighted_tsv.sql +28 -0
- brain/migrations/010_interactions.sql +30 -0
- brain/migrations/011_documents_summary.sql +23 -0
- brain/migrations/012_graphrag.sql +171 -0
- brain/migrations/013_graphrag_communities.sql +125 -0
- brain/migrations/014_graphrag_community_summary_hash.sql +33 -0
- brain/migrations/015_interactions_graph_targets.sql +89 -0
- brain/migrations/016_index_hygiene.sql +61 -0
- brain/migrations/017_elicit.sql +30 -0
- brain/migrations/018_review_gap_signal_kinds.sql +40 -0
- brain/migrations/019_search_queries.sql +35 -0
- brain/migrations/020_link_suggestions.sql +40 -0
- brain/migrations/021_timeline_doc_date.sql +34 -0
- brain/migrations/022_link_suggestions_undirected.sql +84 -0
- brain/migrations/023_search_queries_fts_count.sql +28 -0
- brain/quartz_overrides/__init__.py +8 -0
- brain/quartz_overrides/quartz/bootstrap-cli.mjs +65 -0
- brain/quartz_overrides/quartz/build.ts +568 -0
- brain/quartz_overrides/quartz/cli/args.js +152 -0
- brain/quartz_overrides/quartz/cli/build_partial_handler.js +544 -0
- brain/quartz_overrides/quartz/cli/handlers.js +636 -0
- brain/quartz_overrides/quartz/components/CommandPalette.tsx +172 -0
- brain/quartz_overrides/quartz/components/Explorer.tsx +198 -0
- brain/quartz_overrides/quartz/components/Footer.tsx +27 -0
- brain/quartz_overrides/quartz/components/Graph.tsx +468 -0
- brain/quartz_overrides/quartz/components/PageTitle.tsx +72 -0
- brain/quartz_overrides/quartz/components/RelatedDocs.tsx +38 -0
- brain/quartz_overrides/quartz/components/Search.tsx +161 -0
- brain/quartz_overrides/quartz/components/SummaryLede.tsx +72 -0
- brain/quartz_overrides/quartz/components/index.ts +92 -0
- brain/quartz_overrides/quartz/components/pages/TagContent.tsx +272 -0
- brain/quartz_overrides/quartz/components/scripts/commandPalette.inline.ts +665 -0
- brain/quartz_overrides/quartz/components/scripts/explorer.inline.ts +768 -0
- brain/quartz_overrides/quartz/components/scripts/graph.inline.ts +2302 -0
- brain/quartz_overrides/quartz/components/scripts/relatedDocs.inline.ts +163 -0
- brain/quartz_overrides/quartz/components/scripts/search.inline.ts +1011 -0
- brain/quartz_overrides/quartz/plugins/emitters/contentIndex.ts +546 -0
- brain/quartz_overrides/quartz/plugins/transformers/codeCopy.ts +94 -0
- brain/quartz_overrides/quartz/plugins/transformers/derivedFenceMark.ts +302 -0
- brain/quartz_overrides/quartz/plugins/transformers/emailThread.ts +148 -0
- brain/quartz_overrides/quartz/plugins/transformers/emptyDoorFilter.ts +213 -0
- brain/quartz_overrides/quartz/plugins/transformers/index.ts +114 -0
- brain/quartz_overrides/quartz/plugins/transformers/linkKindMark.ts +205 -0
- brain/quartz_overrides/quartz/plugins/transformers/linkSourceTag.ts +104 -0
- brain/quartz_overrides/quartz/plugins/transformers/relativeDate.ts +100 -0
- brain/quartz_overrides/quartz/plugins/transformers/reloadSignal.ts +131 -0
- brain/quartz_overrides/quartz/processors/parse.ts +371 -0
- brain/quartz_overrides/quartz/processors/parser_cache.ts +78 -0
- brain/quartz_overrides/quartz/static/brain-logo-dark.png +0 -0
- brain/quartz_overrides/quartz/static/brain-logo-light.png +0 -0
- brain/quartz_overrides/quartz/static/codeCopy.js +196 -0
- brain/quartz_overrides/quartz/static/emailThread.js +334 -0
- brain/quartz_overrides/quartz/static/favicon.ico +0 -0
- brain/quartz_overrides/quartz/static/icon.png +0 -0
- brain/quartz_overrides/quartz/static/linkSourceTag.js +104 -0
- brain/quartz_overrides/quartz/static/relativeDate.js +142 -0
- brain/quartz_overrides/quartz/static/reload.js +168 -0
- brain/quartz_overrides/quartz/styles/brain/_article.scss +252 -0
- brain/quartz_overrides/quartz/styles/brain/_atmosphere.scss +113 -0
- brain/quartz_overrides/quartz/styles/brain/_callouts.scss +180 -0
- brain/quartz_overrides/quartz/styles/brain/_cmdk.scss +7 -0
- brain/quartz_overrides/quartz/styles/brain/_code.scss +208 -0
- brain/quartz_overrides/quartz/styles/brain/_command_palette.scss +369 -0
- brain/quartz_overrides/quartz/styles/brain/_email_thread.scss +228 -0
- brain/quartz_overrides/quartz/styles/brain/_explorer.scss +142 -0
- brain/quartz_overrides/quartz/styles/brain/_home.scss +182 -0
- brain/quartz_overrides/quartz/styles/brain/_links.scss +322 -0
- brain/quartz_overrides/quartz/styles/brain/_marginalia.scss +117 -0
- brain/quartz_overrides/quartz/styles/brain/_motion.scss +175 -0
- brain/quartz_overrides/quartz/styles/brain/_people_hub.scss +100 -0
- brain/quartz_overrides/quartz/styles/brain/_related_docs.scss +137 -0
- brain/quartz_overrides/quartz/styles/brain/_search.scss +252 -0
- brain/quartz_overrides/quartz/styles/brain/_sidebar.scss +468 -0
- brain/quartz_overrides/quartz/styles/brain/_summary_lede.scss +56 -0
- brain/quartz_overrides/quartz/styles/brain/_surface.scss +43 -0
- brain/quartz_overrides/quartz/styles/brain/_tag_content.scss +118 -0
- brain/quartz_overrides/quartz/styles/brain/_tokens.scss +197 -0
- brain/quartz_overrides/quartz/styles/brain/_typography.scss +92 -0
- brain/quartz_overrides/quartz/styles/custom.scss +89 -0
- brain/quartz_overrides/quartz/styles/graph.scss +505 -0
- brain/quartz_overrides/quartz/util/ctx.ts +92 -0
- brain/quartz_overrides/quartz/util/fastpath_manifest.ts +608 -0
- brain/quartz_overrides/quartz/util/path.ts +358 -0
- brain/quartz_overrides/quartz/util/sourceIcons.ts +55 -0
- brain/quartz_overrides/quartz.config.ts +270 -0
- brain/quartz_overrides/quartz.layout.ts +314 -0
- brain/queries.py +1188 -0
- brain/rank_fusion.py +8 -0
- brain/resurface.py +210 -0
- brain/review/__init__.py +26 -0
- brain/review/emit.py +27 -0
- brain/review/queries.py +436 -0
- brain/review/render.py +196 -0
- brain/review/scans.py +355 -0
- brain/review/weekly.py +413 -0
- brain/search.py +704 -0
- brain/set_similarity.py +15 -0
- brain/setup.py +1205 -0
- brain/tags.py +56 -0
- brain/templates/Caddyfile.j2 +9 -0
- brain/templates/__init__.py +1 -0
- brain/templates/bin/__init__.py +1 -0
- brain/templates/bin/_brain-brief-fg.sh +25 -0
- brain/templates/bin/_brain-build-fg.sh +53 -0
- brain/templates/bin/_brain-watcher-fg.sh +65 -0
- brain/templates/bin/brain-down.sh +89 -0
- brain/templates/bin/brain-status.sh +83 -0
- brain/templates/bin/brain-up.sh +221 -0
- brain/templates/docker/age/Dockerfile +79 -0
- brain/templates/docker-compose.stock.yml.j2 +26 -0
- brain/templates/docker-compose.yml.j2 +34 -0
- brain/templates/env.example +190 -0
- brain/templates/launchd/__init__.py +1 -0
- brain/templates/launchd/com.brain.brief.plist.j2 +45 -0
- brain/templates/launchd/com.brain.build.plist.j2 +46 -0
- brain/templates/launchd/com.brain.watcher.plist.j2 +46 -0
- brain/templates/skill/SKILL.md +63 -0
- brain/templates/skill/__init__.py +1 -0
- brain/timeline.py +834 -0
- brain/todo.py +124 -0
- brain/uninstall.py +185 -0
- brain/vault/__init__.py +115 -0
- brain/vault/_atomic.py +25 -0
- brain/vault/daily_index.py +228 -0
- brain/vault/derived_links/__init__.py +50 -0
- brain/vault/derived_links/directory.py +683 -0
- brain/vault/derived_links/fence.py +408 -0
- brain/vault/derived_links/gws.py +64 -0
- brain/vault/derived_links/participants.py +143 -0
- brain/vault/derived_links/pass_runner.py +362 -0
- brain/vault/derived_links/rules.py +137 -0
- brain/vault/export.py +683 -0
- brain/vault/frontmatter.py +165 -0
- brain/vault/graph.py +620 -0
- brain/vault/graph_format.py +388 -0
- brain/vault/link_rewrite.py +235 -0
- brain/vault/links.py +260 -0
- brain/vault/note_builder.py +211 -0
- brain/vault/paths.py +55 -0
- brain/vault/quartz_overlay.py +236 -0
- brain/vault/rename.py +591 -0
- brain/vault/resolver.py +304 -0
- brain/vault/slug.py +127 -0
- brain/vault/sync.py +1513 -0
- brain/vault/sync_summaries.py +264 -0
- brain/vault/templates.py +145 -0
- brain/vault/watch.py +1052 -0
- brain/wiki/__init__.py +6 -0
- brain/wiki/_github_slugger.py +76 -0
- brain/wiki/_person_name.py +314 -0
- brain/wiki/build_homepage.py +541 -0
- brain/wiki/build_partial.py +273 -0
- brain/wiki/build_people.py +934 -0
- brain/wiki/build_related.py +758 -0
- brain/wiki/build_swap.py +585 -0
- brain/wiki/build_watcher.py +975 -0
- brain/wiki/edit_classifier.py +215 -0
- brain/wiki/errors.py +10 -0
- brain/wiki/fastpath_manifest.py +475 -0
- brain/wiki/fastpath_state.py +174 -0
- brain/wiki/install.py +296 -0
- brain/wiki/slug.py +111 -0
- secondbrain_py-0.2.1.dist-info/METADATA +195 -0
- secondbrain_py-0.2.1.dist-info/RECORD +273 -0
- secondbrain_py-0.2.1.dist-info/WHEEL +5 -0
- secondbrain_py-0.2.1.dist-info/entry_points.txt +11 -0
- secondbrain_py-0.2.1.dist-info/licenses/LICENSE +21 -0
- secondbrain_py-0.2.1.dist-info/top_level.txt +1 -0
brain/timeline.py
ADDED
|
@@ -0,0 +1,834 @@
|
|
|
1
|
+
"""Temporal evolution of a theme/entity — backs ``brain timeline`` (Plan 05).
|
|
2
|
+
|
|
3
|
+
Buckets entity mentions by document date to show how a theme or entity rose or
|
|
4
|
+
fell over time, with per-bucket co-topics, representative doc titles, and an
|
|
5
|
+
optional best-effort Ollama synthesis line.
|
|
6
|
+
|
|
7
|
+
Single reason to change: the temporal bucketing algorithm. Entity resolution,
|
|
8
|
+
person scoping, co-topic aggregation, and bucket formatting all live here; the
|
|
9
|
+
CLI (`brain.cli`) and MCP (`brain.mcp_server`) only orchestrate.
|
|
10
|
+
|
|
11
|
+
**Relational-only — no Apache AGE / Cypher.** The query joins the migration-012
|
|
12
|
+
relational source-of-truth tables (``graph_entity_mentions``,
|
|
13
|
+
``graph_edge_contributions``, ``graph_entities``) against ``documents``; these
|
|
14
|
+
are pure-SQL tables present even on the stock pgvector image. The temporal
|
|
15
|
+
anchor is ``COALESCE(documents.sent_at, documents.ingested_at)`` (event time
|
|
16
|
+
when known — emails / Krisp — else ingest time), computed inline; when the
|
|
17
|
+
optional migration-021 generated ``doc_date`` column is present it is used
|
|
18
|
+
instead (auto-detected via ``information_schema``).
|
|
19
|
+
"""
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import dataclasses
|
|
23
|
+
import logging
|
|
24
|
+
from collections.abc import Callable
|
|
25
|
+
from dataclasses import dataclass, field
|
|
26
|
+
from datetime import UTC, datetime
|
|
27
|
+
from typing import TYPE_CHECKING, Any
|
|
28
|
+
|
|
29
|
+
import psycopg
|
|
30
|
+
|
|
31
|
+
from .config import (
|
|
32
|
+
_VALID_TIMELINE_GRANULARITIES,
|
|
33
|
+
_VALID_TIMELINE_TRIMS,
|
|
34
|
+
Config,
|
|
35
|
+
)
|
|
36
|
+
from .graph_rag.tenancy import resolve_tenant
|
|
37
|
+
from .queries import resolve_person_to_keys
|
|
38
|
+
|
|
39
|
+
if TYPE_CHECKING:
|
|
40
|
+
from .enrichment import OllamaEnricher
|
|
41
|
+
|
|
42
|
+
_logger = logging.getLogger(__name__)
|
|
43
|
+
|
|
44
|
+
# Number of co-topic entities surfaced per bucket (spec §3f step 3).
|
|
45
|
+
_COTOPIC_LIMIT = 3
|
|
46
|
+
# Entity-resolution fan-out cap — a free-text query that ILIKE-matches more than
|
|
47
|
+
# this many entities is almost certainly too broad; mirrors the
|
|
48
|
+
# ``relational.list_entities`` default LIMIT (spec §3f).
|
|
49
|
+
_ENTITY_RESOLVE_LIMIT = 50
|
|
50
|
+
# Auto-granularity: the coarsest of {year, quarter, month} that yields at least
|
|
51
|
+
# this many non-empty buckets wins; if none clears the bar, fall back to month
|
|
52
|
+
# (so a young/sparse corpus still shows the finest-grained view available).
|
|
53
|
+
_AUTO_MIN_BUCKETS = 3
|
|
54
|
+
# Token budget for the per-bucket DOCUMENT SUMMARIES bundle fed to the synthesis
|
|
55
|
+
# LLM. Caps the prompt so it stays fast on a local 8B model; the bundle is the
|
|
56
|
+
# grounding evidence, so a few hundred tokens of summaries is plenty.
|
|
57
|
+
_SYNTH_DOC_SUMMARY_BUDGET_TOKENS = 1200
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@dataclass(frozen=True)
|
|
61
|
+
class EntityRow:
|
|
62
|
+
"""A resolved ``graph_entities`` row for the timeline seed set.
|
|
63
|
+
|
|
64
|
+
Projected from ``graph_entities`` by :func:`_resolve_entities`. ``id`` is the
|
|
65
|
+
UUID (as text); ``name`` / ``canonical_key`` / ``entity_type`` mirror the
|
|
66
|
+
catalog row. Not the full :class:`brain.graph_rag.schema.GraphEntity` — the
|
|
67
|
+
timeline only needs the identity + display name.
|
|
68
|
+
"""
|
|
69
|
+
|
|
70
|
+
id: str
|
|
71
|
+
name: str
|
|
72
|
+
canonical_key: str
|
|
73
|
+
entity_type: str
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
@dataclass(frozen=True)
|
|
77
|
+
class TimelineBucket:
|
|
78
|
+
"""One time bucket of entity activity (spec §3e).
|
|
79
|
+
|
|
80
|
+
``bucket`` is the human label (e.g. ``"2024-Q1"``); ``bucket_start`` is the
|
|
81
|
+
``date_trunc`` anchor (TZ-aware) used for sorting + JSON. ``doc_count`` is the
|
|
82
|
+
distinct-document count (deduped across the matched entities at the bucket
|
|
83
|
+
level), ``mention_count`` the summed mentions. ``doc_ids`` / ``doc_titles``
|
|
84
|
+
are the bucket's documents; ``cotopics`` the top co-occurring entity names;
|
|
85
|
+
``synthesis`` the optional per-bucket Ollama summary (``None`` unless
|
|
86
|
+
``--synthesize`` ran and succeeded for this bucket).
|
|
87
|
+
"""
|
|
88
|
+
|
|
89
|
+
bucket: str
|
|
90
|
+
bucket_start: datetime
|
|
91
|
+
doc_count: int
|
|
92
|
+
mention_count: int
|
|
93
|
+
doc_ids: list[str] = field(default_factory=list)
|
|
94
|
+
doc_titles: list[str] = field(default_factory=list)
|
|
95
|
+
cotopics: list[str] = field(default_factory=list)
|
|
96
|
+
synthesis: str | None = None
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
@dataclass(frozen=True)
|
|
100
|
+
class TimelineContext:
|
|
101
|
+
"""The wire shape returned by :func:`build_timeline` (CLI ``--json`` / MCP).
|
|
102
|
+
|
|
103
|
+
``query`` is the free-text entity/theme; ``entity_names`` the resolved seed
|
|
104
|
+
entities; ``granularity`` the RESOLVED concrete bucket width (always one of
|
|
105
|
+
``month`` / ``quarter`` / ``year`` — the ``auto`` sentinel is resolved before
|
|
106
|
+
this is built); ``granularity_auto`` is ``True`` when that width was chosen
|
|
107
|
+
automatically (vs. an explicit ``--granularity``). ``person`` is the resolved
|
|
108
|
+
display name when ``--person`` scoped (else ``None``); ``buckets`` the
|
|
109
|
+
ascending time series; ``buckets_omitted`` how many buckets ``--limit``
|
|
110
|
+
trimmed (0 when untrimmed). An all-empty result (no entities / no docs)
|
|
111
|
+
carries an empty ``buckets`` list — never an exception.
|
|
112
|
+
"""
|
|
113
|
+
|
|
114
|
+
query: str
|
|
115
|
+
tenant_id: str
|
|
116
|
+
granularity: str
|
|
117
|
+
granularity_auto: bool = False
|
|
118
|
+
entity_names: list[str] = field(default_factory=list)
|
|
119
|
+
person: str | None = None
|
|
120
|
+
buckets: list[TimelineBucket] = field(default_factory=list)
|
|
121
|
+
buckets_omitted: int = 0
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
# ---------------------------------------------------------------------------
|
|
125
|
+
# Pure helpers (no DB) — unit-tested directly.
|
|
126
|
+
# ---------------------------------------------------------------------------
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _validate_granularity(granularity: str) -> str:
|
|
130
|
+
"""Return ``granularity`` if valid, else raise ``ValueError``.
|
|
131
|
+
|
|
132
|
+
Validated against the same ``{auto, month, quarter, year}`` set the config
|
|
133
|
+
knob uses. ``auto`` is a sentinel resolved to a concrete width by
|
|
134
|
+
:func:`_resolve_auto_granularity` once the matched docs' date span is known;
|
|
135
|
+
the concrete value is later interpolated as the ``date_trunc`` field, so a
|
|
136
|
+
closed whitelist keeps that interpolation safe.
|
|
137
|
+
"""
|
|
138
|
+
normalized = granularity.strip().lower()
|
|
139
|
+
if normalized not in _VALID_TIMELINE_GRANULARITIES:
|
|
140
|
+
raise ValueError(
|
|
141
|
+
"granularity must be one of auto/month/quarter/year "
|
|
142
|
+
f"(got {granularity!r})"
|
|
143
|
+
)
|
|
144
|
+
return normalized
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _resolve_auto_granularity(dates: list[datetime]) -> str:
|
|
148
|
+
"""Pick the auto bucket width for the matched docs' date span (else month).
|
|
149
|
+
|
|
150
|
+
Pure: given the distinct date-anchors, return the COARSEST width
|
|
151
|
+
(iterating ``year`` → ``quarter`` → ``month``) that still produces at least
|
|
152
|
+
:data:`_AUTO_MIN_BUCKETS` non-empty buckets. Coarsest-that-clears-the-bar
|
|
153
|
+
keeps evolution visible (>=3 periods) while minimizing fragmentation — a
|
|
154
|
+
2-year span shows ~8 quarters rather than ~24 months, while a 5-month span
|
|
155
|
+
can only reach the bar at month granularity.
|
|
156
|
+
|
|
157
|
+
Note on ordering: a finer width always yields >= as many buckets as a coarser
|
|
158
|
+
one, so "the finest yielding >=3" would degenerate to always-month and make
|
|
159
|
+
the fallback below unreachable. Coarsest-first is the reading consistent with
|
|
160
|
+
the spec's ``{year, quarter, month}`` ordering AND its explicit
|
|
161
|
+
"if even month yields <3, use month" fallback.
|
|
162
|
+
|
|
163
|
+
When NO width clears the bar — a young or sparse corpus, e.g. the original
|
|
164
|
+
~5-month report where quarter collapsed to one bucket — fall back to
|
|
165
|
+
``month`` so the finest-grained (most informative) view is still shown rather
|
|
166
|
+
than collapsing everything into one coarse bucket. An empty ``dates`` list
|
|
167
|
+
yields ``month``.
|
|
168
|
+
"""
|
|
169
|
+
for gran in ("year", "quarter", "month"):
|
|
170
|
+
distinct = {_bucket_label(d, gran) for d in dates}
|
|
171
|
+
if len(distinct) >= _AUTO_MIN_BUCKETS:
|
|
172
|
+
return gran
|
|
173
|
+
return "month"
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def _validate_trim(trim: str) -> str:
|
|
177
|
+
"""Return ``trim`` if valid, else raise ``ValueError`` (oldest|sparsest)."""
|
|
178
|
+
normalized = trim.strip().lower()
|
|
179
|
+
if normalized not in _VALID_TIMELINE_TRIMS:
|
|
180
|
+
raise ValueError(
|
|
181
|
+
f"trim must be one of oldest/sparsest (got {trim!r})"
|
|
182
|
+
)
|
|
183
|
+
return normalized
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _parse_month(value: str, *, field_name: str) -> datetime:
|
|
187
|
+
"""Parse an ``YYYY-MM`` cutoff into a TZ-aware first-of-month ``datetime``.
|
|
188
|
+
|
|
189
|
+
Raises ``ValueError`` (mapped to a CLI ``BadParameter`` / MCP
|
|
190
|
+
``INVALID_PARAMS`` upstream) on any non ``YYYY-MM`` string.
|
|
191
|
+
"""
|
|
192
|
+
raw = value.strip()
|
|
193
|
+
try:
|
|
194
|
+
parsed = datetime.strptime(raw, "%Y-%m")
|
|
195
|
+
except ValueError as exc:
|
|
196
|
+
raise ValueError(
|
|
197
|
+
f"{field_name} must be an ISO month (YYYY-MM); got {value!r}"
|
|
198
|
+
) from exc
|
|
199
|
+
return parsed.replace(day=1, tzinfo=UTC)
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _add_one_month(moment: datetime) -> datetime:
|
|
203
|
+
"""Return the first instant of the month after ``moment`` (TZ preserved).
|
|
204
|
+
|
|
205
|
+
Used to make ``--until YYYY-MM`` inclusive of the named month: the query
|
|
206
|
+
upper bound becomes ``doc_date < first-of-next-month``.
|
|
207
|
+
"""
|
|
208
|
+
if moment.month == 12:
|
|
209
|
+
return moment.replace(year=moment.year + 1, month=1)
|
|
210
|
+
return moment.replace(month=moment.month + 1)
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def _bucket_label(bucket_start: datetime, granularity: str) -> str:
|
|
214
|
+
"""Format a ``date_trunc`` anchor into a human bucket label.
|
|
215
|
+
|
|
216
|
+
``month`` → ``"2024-03"``, ``quarter`` → ``"2024-Q2"``, ``year`` → ``"2024"``.
|
|
217
|
+
"""
|
|
218
|
+
year = bucket_start.year
|
|
219
|
+
if granularity == "year":
|
|
220
|
+
return f"{year:04d}"
|
|
221
|
+
if granularity == "quarter":
|
|
222
|
+
quarter = (bucket_start.month - 1) // 3 + 1
|
|
223
|
+
return f"{year:04d}-Q{quarter}"
|
|
224
|
+
return f"{year:04d}-{bucket_start.month:02d}"
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _trim_buckets(
|
|
228
|
+
buckets: list[TimelineBucket], limit: int, trim: str
|
|
229
|
+
) -> tuple[list[TimelineBucket], int]:
|
|
230
|
+
"""Trim ``buckets`` to ``limit`` per the ``trim`` strategy (spec §3f step 2).
|
|
231
|
+
|
|
232
|
+
``buckets`` must be sorted ascending by ``bucket_start``. ``oldest`` keeps the
|
|
233
|
+
most-recent ``limit`` buckets (drops the earliest); ``sparsest`` drops the
|
|
234
|
+
buckets with the fewest docs (ties broken by oldest-first removal). Returns
|
|
235
|
+
``(kept_sorted_ascending, omitted_count)``. A non-positive ``limit`` is a
|
|
236
|
+
caller bug guarded upstream; here ``limit <= 0`` trims everything.
|
|
237
|
+
"""
|
|
238
|
+
total = len(buckets)
|
|
239
|
+
if total <= limit:
|
|
240
|
+
return list(buckets), 0
|
|
241
|
+
omitted = total - limit
|
|
242
|
+
if trim == "sparsest":
|
|
243
|
+
# Rank by (doc_count asc, bucket_start asc) and drop the first ``omitted``
|
|
244
|
+
# — the fewest-doc buckets, oldest-first on ties.
|
|
245
|
+
by_sparsity = sorted(
|
|
246
|
+
buckets, key=lambda b: (b.doc_count, b.bucket_start)
|
|
247
|
+
)
|
|
248
|
+
keep = {id(b) for b in by_sparsity[omitted:]}
|
|
249
|
+
kept = [b for b in buckets if id(b) in keep]
|
|
250
|
+
else: # "oldest" — keep the most recent ``limit`` (buckets is ascending)
|
|
251
|
+
kept = buckets[omitted:]
|
|
252
|
+
return kept, omitted
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
# ---------------------------------------------------------------------------
|
|
256
|
+
# DB helpers.
|
|
257
|
+
# ---------------------------------------------------------------------------
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def _doc_date_column_exists(conn: psycopg.Connection[Any]) -> bool:
|
|
261
|
+
"""True iff the optional migration-021 ``documents.doc_date`` column exists.
|
|
262
|
+
|
|
263
|
+
Detected via ``information_schema.columns`` so the timeline transparently
|
|
264
|
+
uses the indexed generated column when present and falls back to the inline
|
|
265
|
+
``COALESCE`` expression otherwise (spec §3e / §6 — zero regression before
|
|
266
|
+
migration 021 is applied).
|
|
267
|
+
"""
|
|
268
|
+
row = conn.execute(
|
|
269
|
+
"SELECT 1 FROM information_schema.columns "
|
|
270
|
+
"WHERE table_name = 'documents' AND column_name = 'doc_date'"
|
|
271
|
+
).fetchone()
|
|
272
|
+
return row is not None
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def _doc_date_expr(conn: psycopg.Connection[Any]) -> str:
|
|
276
|
+
"""Return the SQL doc-date expression (internal whitelist, never user input).
|
|
277
|
+
|
|
278
|
+
``d.doc_date`` when migration 021 is applied, else the inline
|
|
279
|
+
``COALESCE(d.sent_at, d.ingested_at)``. The returned string is one of two
|
|
280
|
+
fixed literals — safe to interpolate into the bucketing SQL.
|
|
281
|
+
"""
|
|
282
|
+
if _doc_date_column_exists(conn):
|
|
283
|
+
return "d.doc_date"
|
|
284
|
+
return "COALESCE(d.sent_at, d.ingested_at)"
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def _resolve_entities(
|
|
288
|
+
conn: psycopg.Connection[Any], tenant_id: str, query: str
|
|
289
|
+
) -> list[EntityRow]:
|
|
290
|
+
"""Resolve a free-text query to ``graph_entities`` via ILIKE (spec §3f step 1).
|
|
291
|
+
|
|
292
|
+
A dedicated parameterized ILIKE against ``name`` / ``canonical_key`` (NOT
|
|
293
|
+
``relational.list_entities``, which has no name filter). Both placeholders
|
|
294
|
+
receive ``f"%{query}%"`` — never f-stringed into the SQL. Returns up to
|
|
295
|
+
:data:`_ENTITY_RESOLVE_LIMIT` matches ordered by name; an empty result is a
|
|
296
|
+
normal "no entities found" outcome, not an error.
|
|
297
|
+
"""
|
|
298
|
+
needle = f"%{query.strip()}%"
|
|
299
|
+
rows = conn.execute(
|
|
300
|
+
"SELECT id::text, name, canonical_key, entity_type "
|
|
301
|
+
"FROM graph_entities "
|
|
302
|
+
"WHERE tenant_id = %s AND (canonical_key ILIKE %s OR name ILIKE %s) "
|
|
303
|
+
"ORDER BY name ASC, id ASC "
|
|
304
|
+
"LIMIT %s",
|
|
305
|
+
(tenant_id, needle, needle, _ENTITY_RESOLVE_LIMIT),
|
|
306
|
+
).fetchall()
|
|
307
|
+
return [
|
|
308
|
+
EntityRow(
|
|
309
|
+
id=str(r[0]), name=str(r[1]), canonical_key=str(r[2]), entity_type=str(r[3])
|
|
310
|
+
)
|
|
311
|
+
for r in rows
|
|
312
|
+
]
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def _owner_entity_ids(
|
|
316
|
+
conn: psycopg.Connection[Any], tenant_id: str, owner_participants: frozenset[str]
|
|
317
|
+
) -> list[str]:
|
|
318
|
+
"""Resolve the corpus owner's participant keys to ``person`` entity ids.
|
|
319
|
+
|
|
320
|
+
Mirrors :func:`brain.graph_rag.themes._person_entity_ids` (kept local to
|
|
321
|
+
preserve the timeline module's single responsibility): a person entity's
|
|
322
|
+
``canonical_key`` is the lowercased People-Hub display name, which the owner
|
|
323
|
+
key set carries alongside emails, so the name key matches and email keys
|
|
324
|
+
harmlessly miss. Empty owner set → no ids.
|
|
325
|
+
"""
|
|
326
|
+
keys = sorted(k for k in owner_participants if k)
|
|
327
|
+
if not keys:
|
|
328
|
+
return []
|
|
329
|
+
rows = conn.execute(
|
|
330
|
+
"SELECT id::text FROM graph_entities "
|
|
331
|
+
"WHERE tenant_id = %s AND entity_type = 'person' "
|
|
332
|
+
"AND lower(canonical_key) = ANY(%s)",
|
|
333
|
+
(tenant_id, keys),
|
|
334
|
+
).fetchall()
|
|
335
|
+
return [str(r[0]) for r in rows]
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def _scope_to_person(
|
|
339
|
+
conn: psycopg.Connection[Any],
|
|
340
|
+
tenant_id: str,
|
|
341
|
+
entity_ids: list[str],
|
|
342
|
+
person_keys: list[str],
|
|
343
|
+
) -> list[str]:
|
|
344
|
+
"""Restrict to docs where the person co-appears as a participant (spec §3f).
|
|
345
|
+
|
|
346
|
+
``person_keys`` are ``PersonMatch.keys`` (participant-key strings) — NOT
|
|
347
|
+
graph entity ids. Overlaps them against ``documents.participants`` and
|
|
348
|
+
intersects with the documents that mention any seed entity. Returns the
|
|
349
|
+
qualifying document-id set (as text); an empty set means the person never
|
|
350
|
+
co-appears with the theme.
|
|
351
|
+
|
|
352
|
+
The overlap is **case-insensitive**: ``documents.participants`` is written by
|
|
353
|
+
ingest extractors in source-preserved case (Gmail emits
|
|
354
|
+
``"Alice Doe <alice@example.com>"``) while ``PersonMatch.keys`` are lowercased, so
|
|
355
|
+
a plain ``&&`` would miss every mixed-case stored value. We unnest the array
|
|
356
|
+
and ``lower()`` each element before comparing — the same pattern the hybrid
|
|
357
|
+
search ``--person`` filter uses (:func:`brain.search.hybrid_search`).
|
|
358
|
+
"""
|
|
359
|
+
if not entity_ids or not person_keys:
|
|
360
|
+
return []
|
|
361
|
+
rows = conn.execute(
|
|
362
|
+
"SELECT id::text FROM documents "
|
|
363
|
+
"WHERE EXISTS (SELECT 1 FROM unnest(participants) AS _p "
|
|
364
|
+
" WHERE lower(_p) = ANY(%s::text[])) "
|
|
365
|
+
"AND id = ANY("
|
|
366
|
+
" SELECT document_id FROM graph_entity_mentions "
|
|
367
|
+
" WHERE entity_id = ANY(%s) AND tenant_id = %s"
|
|
368
|
+
")",
|
|
369
|
+
(person_keys, entity_ids, tenant_id),
|
|
370
|
+
).fetchall()
|
|
371
|
+
return [str(r[0]) for r in rows]
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
@dataclass(frozen=True)
|
|
375
|
+
class _RawBucket:
|
|
376
|
+
"""Intermediate bucket row from :func:`_query_buckets` (pre-enrichment)."""
|
|
377
|
+
|
|
378
|
+
bucket_start: datetime
|
|
379
|
+
doc_count: int
|
|
380
|
+
mention_count: int
|
|
381
|
+
doc_ids: list[str]
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
def _compose_doc_filter(
|
|
385
|
+
entity_ids: list[str],
|
|
386
|
+
tenant_id: str,
|
|
387
|
+
*,
|
|
388
|
+
date_expr: str,
|
|
389
|
+
since: datetime | None,
|
|
390
|
+
until: datetime | None,
|
|
391
|
+
doc_scope: list[str] | None,
|
|
392
|
+
) -> tuple[list[str], list[Any]]:
|
|
393
|
+
"""Build the shared parameterized WHERE clauses for the bucketing queries.
|
|
394
|
+
|
|
395
|
+
Both :func:`_query_buckets` (the grouped bucket query) and
|
|
396
|
+
:func:`_distinct_doc_dates` (the auto-granularity probe) constrain the same
|
|
397
|
+
``graph_entity_mentions``↔``documents`` join the same way, so the clause +
|
|
398
|
+
bound-param composition lives here once (DRY). ``date_expr`` is an internal
|
|
399
|
+
whitelist literal; everything else is bound as ``%s`` parameters.
|
|
400
|
+
"""
|
|
401
|
+
where = ["gem.entity_id = ANY(%s)", "gem.tenant_id = %s"]
|
|
402
|
+
params: list[Any] = [entity_ids, tenant_id]
|
|
403
|
+
if since is not None:
|
|
404
|
+
where.append(f"{date_expr} >= %s")
|
|
405
|
+
params.append(since)
|
|
406
|
+
if until is not None:
|
|
407
|
+
where.append(f"{date_expr} < %s")
|
|
408
|
+
params.append(until)
|
|
409
|
+
if doc_scope is not None:
|
|
410
|
+
where.append("d.id = ANY(%s)")
|
|
411
|
+
params.append(doc_scope)
|
|
412
|
+
return where, params
|
|
413
|
+
|
|
414
|
+
|
|
415
|
+
def _distinct_doc_dates(
|
|
416
|
+
conn: psycopg.Connection[Any],
|
|
417
|
+
tenant_id: str,
|
|
418
|
+
entity_ids: list[str],
|
|
419
|
+
*,
|
|
420
|
+
date_expr: str,
|
|
421
|
+
since: datetime | None,
|
|
422
|
+
until: datetime | None,
|
|
423
|
+
doc_scope: list[str] | None,
|
|
424
|
+
) -> list[datetime]:
|
|
425
|
+
"""Distinct document date-anchors for the matched set (auto-granularity probe).
|
|
426
|
+
|
|
427
|
+
Runs the same join + WHERE as :func:`_query_buckets` but selects the distinct
|
|
428
|
+
``date_expr`` values (one per distinct anchor), so :func:`_resolve_auto_granularity`
|
|
429
|
+
can count how many buckets each width would produce before the full grouped
|
|
430
|
+
query runs. NULL anchors are dropped. Only invoked when granularity is
|
|
431
|
+
``auto``.
|
|
432
|
+
"""
|
|
433
|
+
where, params = _compose_doc_filter(
|
|
434
|
+
entity_ids,
|
|
435
|
+
tenant_id,
|
|
436
|
+
date_expr=date_expr,
|
|
437
|
+
since=since,
|
|
438
|
+
until=until,
|
|
439
|
+
doc_scope=doc_scope,
|
|
440
|
+
)
|
|
441
|
+
sql = (
|
|
442
|
+
f"SELECT DISTINCT {date_expr} AS d "
|
|
443
|
+
"FROM graph_entity_mentions gem "
|
|
444
|
+
"JOIN documents d ON d.id = gem.document_id "
|
|
445
|
+
f"WHERE {' AND '.join(where)}"
|
|
446
|
+
)
|
|
447
|
+
rows = conn.execute(sql, params).fetchall()
|
|
448
|
+
return [row[0] for row in rows if row[0] is not None]
|
|
449
|
+
|
|
450
|
+
|
|
451
|
+
def _query_buckets(
|
|
452
|
+
conn: psycopg.Connection[Any],
|
|
453
|
+
tenant_id: str,
|
|
454
|
+
entity_ids: list[str],
|
|
455
|
+
*,
|
|
456
|
+
granularity: str,
|
|
457
|
+
date_expr: str,
|
|
458
|
+
since: datetime | None,
|
|
459
|
+
until: datetime | None,
|
|
460
|
+
doc_scope: list[str] | None,
|
|
461
|
+
) -> list[_RawBucket]:
|
|
462
|
+
"""Run the temporal bucketing query (spec §3e).
|
|
463
|
+
|
|
464
|
+
One query over all ``entity_ids`` with ``COUNT(DISTINCT d.id)`` /
|
|
465
|
+
``SUM(mention_count)`` / ``ARRAY_AGG(DISTINCT d.id)`` so a doc mentioned by
|
|
466
|
+
several matched entities is counted once per bucket. ``date_expr`` is an
|
|
467
|
+
internal whitelist literal; ``granularity`` is a validated ``date_trunc``
|
|
468
|
+
field passed as a bound parameter. ``since`` / ``until`` (inclusive of the
|
|
469
|
+
named month — ``until`` is already first-of-next-month) and ``doc_scope``
|
|
470
|
+
(person filter) compose as extra parameterized clauses.
|
|
471
|
+
"""
|
|
472
|
+
where, params = _compose_doc_filter(
|
|
473
|
+
entity_ids,
|
|
474
|
+
tenant_id,
|
|
475
|
+
date_expr=date_expr,
|
|
476
|
+
since=since,
|
|
477
|
+
until=until,
|
|
478
|
+
doc_scope=doc_scope,
|
|
479
|
+
)
|
|
480
|
+
sql = (
|
|
481
|
+
f"SELECT date_trunc(%s, {date_expr}) AS bucket_start, "
|
|
482
|
+
" COUNT(DISTINCT d.id) AS doc_count, "
|
|
483
|
+
" COALESCE(SUM(gem.mention_count), 0) AS mention_count, "
|
|
484
|
+
" ARRAY_AGG(DISTINCT d.id::text) AS doc_ids "
|
|
485
|
+
"FROM graph_entity_mentions gem "
|
|
486
|
+
"JOIN documents d ON d.id = gem.document_id "
|
|
487
|
+
f"WHERE {' AND '.join(where)} "
|
|
488
|
+
"GROUP BY 1 "
|
|
489
|
+
"ORDER BY 1 ASC"
|
|
490
|
+
)
|
|
491
|
+
rows = conn.execute(sql, [granularity, *params]).fetchall()
|
|
492
|
+
return [
|
|
493
|
+
_RawBucket(
|
|
494
|
+
bucket_start=row[0],
|
|
495
|
+
doc_count=int(row[1]),
|
|
496
|
+
mention_count=int(row[2]),
|
|
497
|
+
doc_ids=[str(d) for d in (row[3] or [])],
|
|
498
|
+
)
|
|
499
|
+
for row in rows
|
|
500
|
+
]
|
|
501
|
+
|
|
502
|
+
|
|
503
|
+
def _top_cotopics(
|
|
504
|
+
conn: psycopg.Connection[Any],
|
|
505
|
+
tenant_id: str,
|
|
506
|
+
doc_ids: list[str],
|
|
507
|
+
seed_entity_ids: list[str],
|
|
508
|
+
owner_entity_ids: list[str],
|
|
509
|
+
) -> list[str]:
|
|
510
|
+
"""Top co-occurring entity names for a bucket's docs (spec §3f step 3).
|
|
511
|
+
|
|
512
|
+
Aggregates ``graph_edge_contributions`` restricted to the bucket's docs and
|
|
513
|
+
edges touching a seed entity; the non-seed endpoint is the co-topic. Excludes
|
|
514
|
+
the seed entities and the corpus owner (same suppression as themes mode).
|
|
515
|
+
Returns up to :data:`_COTOPIC_LIMIT` names by descending co-occurrence.
|
|
516
|
+
"""
|
|
517
|
+
if not doc_ids or not seed_entity_ids:
|
|
518
|
+
return []
|
|
519
|
+
excluded = seed_entity_ids + owner_entity_ids
|
|
520
|
+
rows = conn.execute(
|
|
521
|
+
"SELECT ge.name, SUM(sub.cooccur) AS total FROM ("
|
|
522
|
+
" SELECT CASE WHEN gec.src_id = ANY(%s) THEN gec.dst_id "
|
|
523
|
+
" ELSE gec.src_id END AS co_id, "
|
|
524
|
+
" gec.cooccur_count AS cooccur "
|
|
525
|
+
" FROM graph_edge_contributions gec "
|
|
526
|
+
" WHERE gec.tenant_id = %s AND gec.document_id = ANY(%s) "
|
|
527
|
+
" AND (gec.src_id = ANY(%s) OR gec.dst_id = ANY(%s)) "
|
|
528
|
+
") sub "
|
|
529
|
+
"JOIN graph_entities ge ON ge.id = sub.co_id AND ge.tenant_id = %s "
|
|
530
|
+
"WHERE sub.co_id::text <> ALL(%s) "
|
|
531
|
+
"GROUP BY ge.name "
|
|
532
|
+
"ORDER BY total DESC, ge.name ASC "
|
|
533
|
+
"LIMIT %s",
|
|
534
|
+
(
|
|
535
|
+
seed_entity_ids,
|
|
536
|
+
tenant_id,
|
|
537
|
+
doc_ids,
|
|
538
|
+
seed_entity_ids,
|
|
539
|
+
seed_entity_ids,
|
|
540
|
+
tenant_id,
|
|
541
|
+
excluded,
|
|
542
|
+
_COTOPIC_LIMIT,
|
|
543
|
+
),
|
|
544
|
+
).fetchall()
|
|
545
|
+
return [str(r[0]) for r in rows]
|
|
546
|
+
|
|
547
|
+
|
|
548
|
+
def _bucket_doc_titles(
|
|
549
|
+
conn: psycopg.Connection[Any], doc_ids: list[str]
|
|
550
|
+
) -> list[str]:
|
|
551
|
+
"""Project document titles for a bucket (spec §3f step 4).
|
|
552
|
+
|
|
553
|
+
Ordered most-recent-first (by the inline doc-date anchor) then title for
|
|
554
|
+
determinism. ``doc_ids`` come straight from :func:`_query_buckets`.
|
|
555
|
+
"""
|
|
556
|
+
if not doc_ids:
|
|
557
|
+
return []
|
|
558
|
+
rows = conn.execute(
|
|
559
|
+
"SELECT title FROM documents WHERE id = ANY(%s) "
|
|
560
|
+
"ORDER BY COALESCE(sent_at, ingested_at) DESC, title ASC",
|
|
561
|
+
(doc_ids,),
|
|
562
|
+
).fetchall()
|
|
563
|
+
return [str(r[0]) for r in rows]
|
|
564
|
+
|
|
565
|
+
|
|
566
|
+
def _bucket_doc_summaries(
|
|
567
|
+
conn: psycopg.Connection[Any], doc_ids: list[str]
|
|
568
|
+
) -> list[tuple[str, str | None]]:
|
|
569
|
+
"""Project ``(title, summary)`` pairs for a bucket's docs (synthesis grounding).
|
|
570
|
+
|
|
571
|
+
Selects ``title`` + ``summary`` ONLY — never ``content`` — mirroring the
|
|
572
|
+
audio bundle's summary-only privacy projection
|
|
573
|
+
(:func:`brain.queries.fetch_document_summary`). Ordered most-recent-first by
|
|
574
|
+
the inline doc-date anchor so the budget keeps the freshest summaries when it
|
|
575
|
+
trims. ``summary`` is ``None`` for a not-yet-enriched doc; the budgeter falls
|
|
576
|
+
back to the title.
|
|
577
|
+
"""
|
|
578
|
+
if not doc_ids:
|
|
579
|
+
return []
|
|
580
|
+
rows = conn.execute(
|
|
581
|
+
"SELECT title, summary FROM documents WHERE id = ANY(%s) "
|
|
582
|
+
"ORDER BY COALESCE(sent_at, ingested_at) DESC, title ASC",
|
|
583
|
+
(doc_ids,),
|
|
584
|
+
).fetchall()
|
|
585
|
+
return [(str(r[0]), r[1]) for r in rows]
|
|
586
|
+
|
|
587
|
+
|
|
588
|
+
def _budget_doc_summaries(
|
|
589
|
+
rows: list[tuple[str, str | None]],
|
|
590
|
+
*,
|
|
591
|
+
count_tokens: Callable[[str], int],
|
|
592
|
+
max_tokens: int,
|
|
593
|
+
) -> list[str]:
|
|
594
|
+
"""Greedily project ``(title, summary)`` rows into a token-budgeted bundle.
|
|
595
|
+
|
|
596
|
+
Each row becomes its ``summary`` (the grounding evidence) or, when the
|
|
597
|
+
summary is NULL/blank, its ``title`` (the documented fallback). Rows are
|
|
598
|
+
included most-recent-first while the running total stays within
|
|
599
|
+
``max_tokens`` (measured by the injected ``count_tokens`` so the budget is
|
|
600
|
+
testable without tiktoken); the first row is always included even if it alone
|
|
601
|
+
exceeds the budget, so the bundle is never empty when docs exist. Mirrors
|
|
602
|
+
:func:`brain.audio.build_prompt`'s greedy budgeting pattern.
|
|
603
|
+
"""
|
|
604
|
+
out: list[str] = []
|
|
605
|
+
used = 0
|
|
606
|
+
for title, summary in rows:
|
|
607
|
+
text = summary.strip() if summary and summary.strip() else title
|
|
608
|
+
cost = count_tokens(text)
|
|
609
|
+
if out and used + cost > max_tokens:
|
|
610
|
+
break
|
|
611
|
+
out.append(text)
|
|
612
|
+
used += cost
|
|
613
|
+
return out
|
|
614
|
+
|
|
615
|
+
|
|
616
|
+
def _synthesize_buckets(
|
|
617
|
+
buckets: list[TimelineBucket],
|
|
618
|
+
*,
|
|
619
|
+
entity_name: str,
|
|
620
|
+
enricher: OllamaEnricher,
|
|
621
|
+
synth_limit: int,
|
|
622
|
+
fetch_summaries: Callable[[list[str]], list[str]],
|
|
623
|
+
) -> list[TimelineBucket]:
|
|
624
|
+
"""Attach best-effort, content-grounded Ollama summaries (spec §3f step 5).
|
|
625
|
+
|
|
626
|
+
Synthesizes the top ``synth_limit`` buckets by ``doc_count`` (ties broken by
|
|
627
|
+
most-recent ``bucket_start``), but iterates the buckets OLDEST→NEWEST so each
|
|
628
|
+
synthesized bucket can be handed the chronologically previous bucket's label,
|
|
629
|
+
co-topics, and (when that previous bucket was itself synthesized) its
|
|
630
|
+
already-generated synthesis — enabling the model to state what CHANGED. Each
|
|
631
|
+
chosen bucket's document summaries are fetched + token-budgeted via
|
|
632
|
+
``fetch_summaries`` (injected so this stays unit-testable without a DB) and
|
|
633
|
+
passed as the model's primary grounding evidence.
|
|
634
|
+
|
|
635
|
+
Best-effort: ``summarize_bucket`` never raises, returning ``None`` on Ollama
|
|
636
|
+
failure, so the timeline still renders. Returns a new bucket list
|
|
637
|
+
(immutability) preserving the original ascending order.
|
|
638
|
+
"""
|
|
639
|
+
if synth_limit <= 0 or not buckets:
|
|
640
|
+
return buckets
|
|
641
|
+
densest = sorted(
|
|
642
|
+
buckets, key=lambda b: (b.doc_count, b.bucket_start), reverse=True
|
|
643
|
+
)[:synth_limit]
|
|
644
|
+
chosen = {id(b) for b in densest}
|
|
645
|
+
out: list[TimelineBucket] = []
|
|
646
|
+
prev: TimelineBucket | None = None
|
|
647
|
+
for bucket in buckets: # ascending (oldest → newest)
|
|
648
|
+
if id(bucket) in chosen:
|
|
649
|
+
doc_summaries = fetch_summaries(bucket.doc_ids)
|
|
650
|
+
summary = enricher.summarize_bucket(
|
|
651
|
+
bucket_label=bucket.bucket,
|
|
652
|
+
entity_name=entity_name,
|
|
653
|
+
doc_titles=bucket.doc_titles,
|
|
654
|
+
cotopics=bucket.cotopics,
|
|
655
|
+
doc_summaries=doc_summaries,
|
|
656
|
+
prev_bucket_label=prev.bucket if prev else None,
|
|
657
|
+
prev_cotopics=prev.cotopics if prev else None,
|
|
658
|
+
prev_synthesis=prev.synthesis if prev else None,
|
|
659
|
+
)
|
|
660
|
+
new_bucket = dataclasses.replace(bucket, synthesis=summary)
|
|
661
|
+
out.append(new_bucket)
|
|
662
|
+
prev = new_bucket
|
|
663
|
+
else:
|
|
664
|
+
out.append(bucket)
|
|
665
|
+
prev = bucket
|
|
666
|
+
return out
|
|
667
|
+
|
|
668
|
+
|
|
669
|
+
def build_timeline(
|
|
670
|
+
conn: psycopg.Connection[Any],
|
|
671
|
+
cfg: Config,
|
|
672
|
+
query: str,
|
|
673
|
+
*,
|
|
674
|
+
granularity: str | None = None,
|
|
675
|
+
since: str | None = None,
|
|
676
|
+
until: str | None = None,
|
|
677
|
+
limit: int | None = None,
|
|
678
|
+
person: str | None = None,
|
|
679
|
+
synthesize: bool = False,
|
|
680
|
+
enricher: OllamaEnricher | None = None,
|
|
681
|
+
tenant: str | None = None,
|
|
682
|
+
) -> TimelineContext:
|
|
683
|
+
"""Build the temporal timeline for ``query`` (spec §3f — the orchestrator).
|
|
684
|
+
|
|
685
|
+
Resolves the entity/theme to ``graph_entities`` (ILIKE), optionally scopes to
|
|
686
|
+
a ``--person``'s co-documents, picks the bucket width (``auto`` — the default
|
|
687
|
+
— chooses the coarsest of {year, quarter, month} yielding >=3 buckets, else
|
|
688
|
+
month; an explicit value forces it), buckets mentions by document date,
|
|
689
|
+
attaches per-bucket co-topics + doc titles, trims to ``limit``
|
|
690
|
+
(``BRAIN_TIMELINE_TRIM`` end), and — when ``synthesize`` and an ``enricher``
|
|
691
|
+
are supplied — adds a best-effort, content-grounded summary to the densest
|
|
692
|
+
buckets (the docs' summaries feed the LLM, not just their titles).
|
|
693
|
+
|
|
694
|
+
Graceful, never-raising on data shape: zero matching entities or zero docs
|
|
695
|
+
yields an empty :class:`TimelineContext`. Only programmer/usage errors raise:
|
|
696
|
+
a bad ``granularity`` / ``since`` / ``until`` raises ``ValueError``; an
|
|
697
|
+
unknown / ambiguous ``person`` propagates ``PersonNotFound`` /
|
|
698
|
+
``PersonAmbiguous`` (mapped to clean CLI/MCP errors by the caller).
|
|
699
|
+
"""
|
|
700
|
+
requested = _validate_granularity(granularity or cfg.timeline_granularity)
|
|
701
|
+
auto = requested == "auto"
|
|
702
|
+
# Concrete width for the empty-data early returns (auto with no docs → month).
|
|
703
|
+
empty_gran = "month" if auto else requested
|
|
704
|
+
trim = _validate_trim(cfg.timeline_trim)
|
|
705
|
+
limit_n = cfg.timeline_limit if limit is None else limit
|
|
706
|
+
if limit_n < 1:
|
|
707
|
+
raise ValueError(f"limit must be a positive integer (got {limit_n})")
|
|
708
|
+
tenant_id = resolve_tenant(cfg, tenant)
|
|
709
|
+
since_dt = _parse_month(since, field_name="since") if since else None
|
|
710
|
+
# ``until`` is inclusive of the named month: upper bound = first-of-next-month.
|
|
711
|
+
until_dt = (
|
|
712
|
+
_add_one_month(_parse_month(until, field_name="until")) if until else None
|
|
713
|
+
)
|
|
714
|
+
|
|
715
|
+
entities = _resolve_entities(conn, tenant_id, query)
|
|
716
|
+
if not entities:
|
|
717
|
+
return TimelineContext(
|
|
718
|
+
query=query,
|
|
719
|
+
tenant_id=tenant_id,
|
|
720
|
+
granularity=empty_gran,
|
|
721
|
+
granularity_auto=auto,
|
|
722
|
+
)
|
|
723
|
+
|
|
724
|
+
entity_ids = [e.id for e in entities]
|
|
725
|
+
entity_names = [e.name for e in entities]
|
|
726
|
+
|
|
727
|
+
person_display: str | None = None
|
|
728
|
+
doc_scope: list[str] | None = None
|
|
729
|
+
if person is not None:
|
|
730
|
+
match = resolve_person_to_keys(conn, person)
|
|
731
|
+
person_display = match.display_name
|
|
732
|
+
doc_scope = _scope_to_person(conn, tenant_id, entity_ids, match.keys)
|
|
733
|
+
if not doc_scope:
|
|
734
|
+
_logger.warning(
|
|
735
|
+
"timeline: no documents where %r co-appears as a participant",
|
|
736
|
+
person,
|
|
737
|
+
)
|
|
738
|
+
return TimelineContext(
|
|
739
|
+
query=query,
|
|
740
|
+
tenant_id=tenant_id,
|
|
741
|
+
granularity=empty_gran,
|
|
742
|
+
granularity_auto=auto,
|
|
743
|
+
entity_names=entity_names,
|
|
744
|
+
person=person_display,
|
|
745
|
+
)
|
|
746
|
+
|
|
747
|
+
date_expr = _doc_date_expr(conn)
|
|
748
|
+
# Auto-granularity: probe the matched docs' distinct date-anchors and pick the
|
|
749
|
+
# coarsest width yielding >=3 buckets (else month) BEFORE the grouped query.
|
|
750
|
+
if auto:
|
|
751
|
+
dates = _distinct_doc_dates(
|
|
752
|
+
conn,
|
|
753
|
+
tenant_id,
|
|
754
|
+
entity_ids,
|
|
755
|
+
date_expr=date_expr,
|
|
756
|
+
since=since_dt,
|
|
757
|
+
until=until_dt,
|
|
758
|
+
doc_scope=doc_scope,
|
|
759
|
+
)
|
|
760
|
+
gran = _resolve_auto_granularity(dates)
|
|
761
|
+
else:
|
|
762
|
+
gran = requested
|
|
763
|
+
|
|
764
|
+
raw_buckets = _query_buckets(
|
|
765
|
+
conn,
|
|
766
|
+
tenant_id,
|
|
767
|
+
entity_ids,
|
|
768
|
+
granularity=gran,
|
|
769
|
+
date_expr=date_expr,
|
|
770
|
+
since=since_dt,
|
|
771
|
+
until=until_dt,
|
|
772
|
+
doc_scope=doc_scope,
|
|
773
|
+
)
|
|
774
|
+
if not raw_buckets:
|
|
775
|
+
return TimelineContext(
|
|
776
|
+
query=query,
|
|
777
|
+
tenant_id=tenant_id,
|
|
778
|
+
granularity=gran,
|
|
779
|
+
granularity_auto=auto,
|
|
780
|
+
entity_names=entity_names,
|
|
781
|
+
person=person_display,
|
|
782
|
+
)
|
|
783
|
+
|
|
784
|
+
owner_ids = _owner_entity_ids(conn, tenant_id, cfg.owner_participants)
|
|
785
|
+
enriched: list[TimelineBucket] = []
|
|
786
|
+
for raw in raw_buckets:
|
|
787
|
+
cotopics = _top_cotopics(conn, tenant_id, raw.doc_ids, entity_ids, owner_ids)
|
|
788
|
+
titles = _bucket_doc_titles(conn, raw.doc_ids)
|
|
789
|
+
enriched.append(
|
|
790
|
+
TimelineBucket(
|
|
791
|
+
bucket=_bucket_label(raw.bucket_start, gran),
|
|
792
|
+
bucket_start=raw.bucket_start,
|
|
793
|
+
doc_count=raw.doc_count,
|
|
794
|
+
mention_count=raw.mention_count,
|
|
795
|
+
doc_ids=raw.doc_ids,
|
|
796
|
+
doc_titles=titles,
|
|
797
|
+
cotopics=cotopics,
|
|
798
|
+
)
|
|
799
|
+
)
|
|
800
|
+
|
|
801
|
+
kept, omitted = _trim_buckets(enriched, limit_n, trim)
|
|
802
|
+
|
|
803
|
+
if synthesize and enricher is not None and cfg.timeline_synth_limit > 0:
|
|
804
|
+
# Grounding evidence: the docs' SUMMARIES (title fallback), token-budgeted
|
|
805
|
+
# so the prompt stays fast on a local 8B model. Injected as a closure so
|
|
806
|
+
# ``_synthesize_buckets`` stays unit-testable without a DB.
|
|
807
|
+
token_counter = enricher.count_tokens
|
|
808
|
+
|
|
809
|
+
def _fetch_bucket_summaries(doc_ids: list[str]) -> list[str]:
|
|
810
|
+
rows = _bucket_doc_summaries(conn, doc_ids)
|
|
811
|
+
return _budget_doc_summaries(
|
|
812
|
+
rows,
|
|
813
|
+
count_tokens=token_counter,
|
|
814
|
+
max_tokens=_SYNTH_DOC_SUMMARY_BUDGET_TOKENS,
|
|
815
|
+
)
|
|
816
|
+
|
|
817
|
+
kept = _synthesize_buckets(
|
|
818
|
+
kept,
|
|
819
|
+
entity_name=entity_names[0],
|
|
820
|
+
enricher=enricher,
|
|
821
|
+
synth_limit=cfg.timeline_synth_limit,
|
|
822
|
+
fetch_summaries=_fetch_bucket_summaries,
|
|
823
|
+
)
|
|
824
|
+
|
|
825
|
+
return TimelineContext(
|
|
826
|
+
query=query,
|
|
827
|
+
tenant_id=tenant_id,
|
|
828
|
+
granularity=gran,
|
|
829
|
+
granularity_auto=auto,
|
|
830
|
+
entity_names=entity_names,
|
|
831
|
+
person=person_display,
|
|
832
|
+
buckets=kept,
|
|
833
|
+
buckets_omitted=omitted,
|
|
834
|
+
)
|