secondbrain-py 0.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- brain/__init__.py +0 -0
- brain/__main__.py +18 -0
- brain/_capture_command.py +445 -0
- brain/_compose.py +52 -0
- brain/activity.py +206 -0
- brain/ask.py +631 -0
- brain/audio.py +591 -0
- brain/backfill/__init__.py +12 -0
- brain/backfill/search_extras.py +141 -0
- brain/backfill/source_rows.py +101 -0
- brain/bin/__init__.py +1 -0
- brain/bin/_launcher.py +107 -0
- brain/bin/down.py +8 -0
- brain/bin/launchd.py +268 -0
- brain/bin/monitor.py +570 -0
- brain/bin/rebuild.py +8 -0
- brain/bin/status.py +8 -0
- brain/bin/up.py +8 -0
- brain/brief.py +272 -0
- brain/capture.py +49 -0
- brain/chat.py +293 -0
- brain/cli.py +9760 -0
- brain/cli_claude.py +81 -0
- brain/cli_connect.py +285 -0
- brain/cli_demo.py +266 -0
- brain/config.py +1949 -0
- brain/connect.py +925 -0
- brain/db.py +540 -0
- brain/demo/__init__.py +452 -0
- brain/demo/corpus/manifest.json +403 -0
- brain/demo/embedder.py +74 -0
- brain/durations.py +84 -0
- brain/edit_session.py +156 -0
- brain/editor.py +67 -0
- brain/elicit/__init__.py +16 -0
- brain/elicit/detectors.py +250 -0
- brain/elicit/drafter.py +70 -0
- brain/elicit/queue.py +220 -0
- brain/elicit/schema.py +48 -0
- brain/elicit/session.py +445 -0
- brain/embedding_targets.py +54 -0
- brain/embeddings.py +424 -0
- brain/enrichment.py +808 -0
- brain/errors.py +357 -0
- brain/eval/__init__.py +129 -0
- brain/eval/answer_eval.py +281 -0
- brain/eval/baseline.py +265 -0
- brain/eval/concept_extraction.py +378 -0
- brain/eval/corpus.py +152 -0
- brain/eval/errors.py +19 -0
- brain/eval/graph_baseline.py +226 -0
- brain/eval/graph_retrieval.py +202 -0
- brain/eval/graph_runner.py +319 -0
- brain/eval/metrics.py +101 -0
- brain/eval/runner.py +223 -0
- brain/format.py +783 -0
- brain/gaps.py +390 -0
- brain/graph_rag/__init__.py +94 -0
- brain/graph_rag/_retrieval_common.py +113 -0
- brain/graph_rag/aggregates.py +303 -0
- brain/graph_rag/aliases/__init__.py +583 -0
- brain/graph_rag/backends/__init__.py +10 -0
- brain/graph_rag/backends/_age_helpers.py +473 -0
- brain/graph_rag/backends/age.py +782 -0
- brain/graph_rag/backends/base.py +272 -0
- brain/graph_rag/build.py +344 -0
- brain/graph_rag/communities.py +644 -0
- brain/graph_rag/communities_summary.py +437 -0
- brain/graph_rag/concepts.py +202 -0
- brain/graph_rag/cooccur.py +193 -0
- brain/graph_rag/cross_type.py +312 -0
- brain/graph_rag/extract.py +885 -0
- brain/graph_rag/fuse.py +371 -0
- brain/graph_rag/global_.py +412 -0
- brain/graph_rag/grouping.py +372 -0
- brain/graph_rag/person_resolver.py +167 -0
- brain/graph_rag/reconcile.py +792 -0
- brain/graph_rag/relational.py +353 -0
- brain/graph_rag/retrieve.py +526 -0
- brain/graph_rag/router.py +288 -0
- brain/graph_rag/schema.py +320 -0
- brain/graph_rag/sync.py +237 -0
- brain/graph_rag/tenancy.py +43 -0
- brain/graph_rag/themes.py +501 -0
- brain/graph_rag/weighting.py +202 -0
- brain/ingest/__init__.py +1926 -0
- brain/ingest/chunker.py +249 -0
- brain/ingest/docx.py +40 -0
- brain/ingest/gmail.py +621 -0
- brain/ingest/markdown.py +37 -0
- brain/ingest/pdf.py +61 -0
- brain/ingest/stdin.py +22 -0
- brain/ingest/sub_tokens.py +91 -0
- brain/ingest/text.py +16 -0
- brain/interactions.py +205 -0
- brain/maintenance.py +355 -0
- brain/mcp_server.py +3405 -0
- brain/migrations/001_init.sql +43 -0
- brain/migrations/002_qwen3_embedding.sql +17 -0
- brain/migrations/003_vault_model.sql +41 -0
- brain/migrations/004_relax_content_hash_uniqueness.sql +18 -0
- brain/migrations/005_derived_links.sql +67 -0
- brain/migrations/006_dedup_file_by_source_path.sql +25 -0
- brain/migrations/007_email_thread_and_draft.sql +15 -0
- brain/migrations/008_gmail_thread_unique.sql +11 -0
- brain/migrations/009_chunks_weighted_tsv.sql +28 -0
- brain/migrations/010_interactions.sql +30 -0
- brain/migrations/011_documents_summary.sql +23 -0
- brain/migrations/012_graphrag.sql +171 -0
- brain/migrations/013_graphrag_communities.sql +125 -0
- brain/migrations/014_graphrag_community_summary_hash.sql +33 -0
- brain/migrations/015_interactions_graph_targets.sql +89 -0
- brain/migrations/016_index_hygiene.sql +61 -0
- brain/migrations/017_elicit.sql +30 -0
- brain/migrations/018_review_gap_signal_kinds.sql +40 -0
- brain/migrations/019_search_queries.sql +35 -0
- brain/migrations/020_link_suggestions.sql +40 -0
- brain/migrations/021_timeline_doc_date.sql +34 -0
- brain/migrations/022_link_suggestions_undirected.sql +84 -0
- brain/migrations/023_search_queries_fts_count.sql +28 -0
- brain/quartz_overrides/__init__.py +8 -0
- brain/quartz_overrides/quartz/bootstrap-cli.mjs +65 -0
- brain/quartz_overrides/quartz/build.ts +568 -0
- brain/quartz_overrides/quartz/cli/args.js +152 -0
- brain/quartz_overrides/quartz/cli/build_partial_handler.js +544 -0
- brain/quartz_overrides/quartz/cli/handlers.js +636 -0
- brain/quartz_overrides/quartz/components/CommandPalette.tsx +172 -0
- brain/quartz_overrides/quartz/components/Explorer.tsx +198 -0
- brain/quartz_overrides/quartz/components/Footer.tsx +27 -0
- brain/quartz_overrides/quartz/components/Graph.tsx +468 -0
- brain/quartz_overrides/quartz/components/PageTitle.tsx +72 -0
- brain/quartz_overrides/quartz/components/RelatedDocs.tsx +38 -0
- brain/quartz_overrides/quartz/components/Search.tsx +161 -0
- brain/quartz_overrides/quartz/components/SummaryLede.tsx +72 -0
- brain/quartz_overrides/quartz/components/index.ts +92 -0
- brain/quartz_overrides/quartz/components/pages/TagContent.tsx +272 -0
- brain/quartz_overrides/quartz/components/scripts/commandPalette.inline.ts +665 -0
- brain/quartz_overrides/quartz/components/scripts/explorer.inline.ts +768 -0
- brain/quartz_overrides/quartz/components/scripts/graph.inline.ts +2302 -0
- brain/quartz_overrides/quartz/components/scripts/relatedDocs.inline.ts +163 -0
- brain/quartz_overrides/quartz/components/scripts/search.inline.ts +1011 -0
- brain/quartz_overrides/quartz/plugins/emitters/contentIndex.ts +546 -0
- brain/quartz_overrides/quartz/plugins/transformers/codeCopy.ts +94 -0
- brain/quartz_overrides/quartz/plugins/transformers/derivedFenceMark.ts +302 -0
- brain/quartz_overrides/quartz/plugins/transformers/emailThread.ts +148 -0
- brain/quartz_overrides/quartz/plugins/transformers/emptyDoorFilter.ts +213 -0
- brain/quartz_overrides/quartz/plugins/transformers/index.ts +114 -0
- brain/quartz_overrides/quartz/plugins/transformers/linkKindMark.ts +205 -0
- brain/quartz_overrides/quartz/plugins/transformers/linkSourceTag.ts +104 -0
- brain/quartz_overrides/quartz/plugins/transformers/relativeDate.ts +100 -0
- brain/quartz_overrides/quartz/plugins/transformers/reloadSignal.ts +131 -0
- brain/quartz_overrides/quartz/processors/parse.ts +371 -0
- brain/quartz_overrides/quartz/processors/parser_cache.ts +78 -0
- brain/quartz_overrides/quartz/static/brain-logo-dark.png +0 -0
- brain/quartz_overrides/quartz/static/brain-logo-light.png +0 -0
- brain/quartz_overrides/quartz/static/codeCopy.js +196 -0
- brain/quartz_overrides/quartz/static/emailThread.js +334 -0
- brain/quartz_overrides/quartz/static/favicon.ico +0 -0
- brain/quartz_overrides/quartz/static/icon.png +0 -0
- brain/quartz_overrides/quartz/static/linkSourceTag.js +104 -0
- brain/quartz_overrides/quartz/static/relativeDate.js +142 -0
- brain/quartz_overrides/quartz/static/reload.js +168 -0
- brain/quartz_overrides/quartz/styles/brain/_article.scss +252 -0
- brain/quartz_overrides/quartz/styles/brain/_atmosphere.scss +113 -0
- brain/quartz_overrides/quartz/styles/brain/_callouts.scss +180 -0
- brain/quartz_overrides/quartz/styles/brain/_cmdk.scss +7 -0
- brain/quartz_overrides/quartz/styles/brain/_code.scss +208 -0
- brain/quartz_overrides/quartz/styles/brain/_command_palette.scss +369 -0
- brain/quartz_overrides/quartz/styles/brain/_email_thread.scss +228 -0
- brain/quartz_overrides/quartz/styles/brain/_explorer.scss +142 -0
- brain/quartz_overrides/quartz/styles/brain/_home.scss +182 -0
- brain/quartz_overrides/quartz/styles/brain/_links.scss +322 -0
- brain/quartz_overrides/quartz/styles/brain/_marginalia.scss +117 -0
- brain/quartz_overrides/quartz/styles/brain/_motion.scss +175 -0
- brain/quartz_overrides/quartz/styles/brain/_people_hub.scss +100 -0
- brain/quartz_overrides/quartz/styles/brain/_related_docs.scss +137 -0
- brain/quartz_overrides/quartz/styles/brain/_search.scss +252 -0
- brain/quartz_overrides/quartz/styles/brain/_sidebar.scss +468 -0
- brain/quartz_overrides/quartz/styles/brain/_summary_lede.scss +56 -0
- brain/quartz_overrides/quartz/styles/brain/_surface.scss +43 -0
- brain/quartz_overrides/quartz/styles/brain/_tag_content.scss +118 -0
- brain/quartz_overrides/quartz/styles/brain/_tokens.scss +197 -0
- brain/quartz_overrides/quartz/styles/brain/_typography.scss +92 -0
- brain/quartz_overrides/quartz/styles/custom.scss +89 -0
- brain/quartz_overrides/quartz/styles/graph.scss +505 -0
- brain/quartz_overrides/quartz/util/ctx.ts +92 -0
- brain/quartz_overrides/quartz/util/fastpath_manifest.ts +608 -0
- brain/quartz_overrides/quartz/util/path.ts +358 -0
- brain/quartz_overrides/quartz/util/sourceIcons.ts +55 -0
- brain/quartz_overrides/quartz.config.ts +270 -0
- brain/quartz_overrides/quartz.layout.ts +314 -0
- brain/queries.py +1188 -0
- brain/rank_fusion.py +8 -0
- brain/resurface.py +210 -0
- brain/review/__init__.py +26 -0
- brain/review/emit.py +27 -0
- brain/review/queries.py +436 -0
- brain/review/render.py +196 -0
- brain/review/scans.py +355 -0
- brain/review/weekly.py +413 -0
- brain/search.py +704 -0
- brain/set_similarity.py +15 -0
- brain/setup.py +1205 -0
- brain/tags.py +56 -0
- brain/templates/Caddyfile.j2 +9 -0
- brain/templates/__init__.py +1 -0
- brain/templates/bin/__init__.py +1 -0
- brain/templates/bin/_brain-brief-fg.sh +25 -0
- brain/templates/bin/_brain-build-fg.sh +53 -0
- brain/templates/bin/_brain-watcher-fg.sh +65 -0
- brain/templates/bin/brain-down.sh +89 -0
- brain/templates/bin/brain-status.sh +83 -0
- brain/templates/bin/brain-up.sh +221 -0
- brain/templates/docker/age/Dockerfile +79 -0
- brain/templates/docker-compose.stock.yml.j2 +26 -0
- brain/templates/docker-compose.yml.j2 +34 -0
- brain/templates/env.example +190 -0
- brain/templates/launchd/__init__.py +1 -0
- brain/templates/launchd/com.brain.brief.plist.j2 +45 -0
- brain/templates/launchd/com.brain.build.plist.j2 +46 -0
- brain/templates/launchd/com.brain.watcher.plist.j2 +46 -0
- brain/templates/skill/SKILL.md +63 -0
- brain/templates/skill/__init__.py +1 -0
- brain/timeline.py +834 -0
- brain/todo.py +124 -0
- brain/uninstall.py +185 -0
- brain/vault/__init__.py +115 -0
- brain/vault/_atomic.py +25 -0
- brain/vault/daily_index.py +228 -0
- brain/vault/derived_links/__init__.py +50 -0
- brain/vault/derived_links/directory.py +683 -0
- brain/vault/derived_links/fence.py +408 -0
- brain/vault/derived_links/gws.py +64 -0
- brain/vault/derived_links/participants.py +143 -0
- brain/vault/derived_links/pass_runner.py +362 -0
- brain/vault/derived_links/rules.py +137 -0
- brain/vault/export.py +683 -0
- brain/vault/frontmatter.py +165 -0
- brain/vault/graph.py +620 -0
- brain/vault/graph_format.py +388 -0
- brain/vault/link_rewrite.py +235 -0
- brain/vault/links.py +260 -0
- brain/vault/note_builder.py +211 -0
- brain/vault/paths.py +55 -0
- brain/vault/quartz_overlay.py +236 -0
- brain/vault/rename.py +591 -0
- brain/vault/resolver.py +304 -0
- brain/vault/slug.py +127 -0
- brain/vault/sync.py +1513 -0
- brain/vault/sync_summaries.py +264 -0
- brain/vault/templates.py +145 -0
- brain/vault/watch.py +1052 -0
- brain/wiki/__init__.py +6 -0
- brain/wiki/_github_slugger.py +76 -0
- brain/wiki/_person_name.py +314 -0
- brain/wiki/build_homepage.py +541 -0
- brain/wiki/build_partial.py +273 -0
- brain/wiki/build_people.py +934 -0
- brain/wiki/build_related.py +758 -0
- brain/wiki/build_swap.py +585 -0
- brain/wiki/build_watcher.py +975 -0
- brain/wiki/edit_classifier.py +215 -0
- brain/wiki/errors.py +10 -0
- brain/wiki/fastpath_manifest.py +475 -0
- brain/wiki/fastpath_state.py +174 -0
- brain/wiki/install.py +296 -0
- brain/wiki/slug.py +111 -0
- secondbrain_py-0.2.1.dist-info/METADATA +195 -0
- secondbrain_py-0.2.1.dist-info/RECORD +273 -0
- secondbrain_py-0.2.1.dist-info/WHEEL +5 -0
- secondbrain_py-0.2.1.dist-info/entry_points.txt +11 -0
- secondbrain_py-0.2.1.dist-info/licenses/LICENSE +21 -0
- secondbrain_py-0.2.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,782 @@
|
|
|
1
|
+
"""Apache AGE implementation of :class:`GraphBackend` (spec §4 D1/D10, §5b, §6a).
|
|
2
|
+
|
|
3
|
+
The graph lives in one AGE graph inside Postgres; this backend translates the
|
|
4
|
+
narrow Protocol into parameterized openCypher. It is the wave-G0 retrieval
|
|
5
|
+
canary — its traversal/scope Cypher was validated against a live AGE 1.5.0
|
|
6
|
+
instance before any later wave depended on it.
|
|
7
|
+
|
|
8
|
+
The instance-state-free helpers (module constants, agtype parsing, input
|
|
9
|
+
validation, the inline property-map builder, the property-index DDL builder, and
|
|
10
|
+
the **relational** ``scope_person`` two-hop) live in
|
|
11
|
+
:mod:`brain.graph_rag.backends._age_helpers` (the G2 wave-boundary file-size
|
|
12
|
+
split); this module owns the ``AgeBackend`` class — session handling, Cypher
|
|
13
|
+
execution, and the Protocol methods — and imports those helpers by their original
|
|
14
|
+
names. The split is behavior-preserving.
|
|
15
|
+
|
|
16
|
+
Empirical AGE behaviour this module is built around (proven on live AGE 1.5.0,
|
|
17
|
+
not assumed):
|
|
18
|
+
|
|
19
|
+
* **Call shape.** ``ag_catalog.cypher('<graph>', $$ <query> $$, <params>) AS
|
|
20
|
+
(<col> ag_catalog.agtype)``. The graph name is a ``name`` constant and the
|
|
21
|
+
query a ``cstring`` constant — *neither* can be a ``%s`` placeholder. Only the
|
|
22
|
+
third ``params`` argument is bound, as ``%s::ag_catalog.agtype`` over a JSON
|
|
23
|
+
string; Cypher references those values as ``$key``. We therefore embed the
|
|
24
|
+
(validated) graph name as a literal and route every dynamic *value* through
|
|
25
|
+
the agtype params map — values are never string-formatted into Cypher.
|
|
26
|
+
* **search_path.** Any non-trivial Cypher (a ``WHERE`` comparison, a ``MERGE``)
|
|
27
|
+
needs the ``ag_catalog`` operators/casts (``=``→boolean, ``@>`` containment).
|
|
28
|
+
We ``SET search_path = ag_catalog, "$user", public`` for the duration of those
|
|
29
|
+
statements and ``RESET`` immediately (see :meth:`_age_session`) — never a
|
|
30
|
+
global leak, matching ``tests/conftest._reset_age_graph`` and
|
|
31
|
+
:func:`brain.db.load_age`. Catalog DDL (``create_graph``/``create_vlabel``)
|
|
32
|
+
and the property-index DDL are fully ``ag_catalog``-qualified and need no
|
|
33
|
+
search_path.
|
|
34
|
+
* **MERGE + SET.** A ``SET`` in the same statement as a ``MERGE`` that *creates*
|
|
35
|
+
the element does **not** persist (and AGE rejects ``ON CREATE SET`` /
|
|
36
|
+
``ON MATCH SET``). So property-bearing upserts are two statements: ``MERGE``
|
|
37
|
+
to get-or-create, then a separate ``MATCH ... SET x += {map}`` that always
|
|
38
|
+
lands on the match path. Aggregate edges are delete-then-``CREATE`` with
|
|
39
|
+
inline properties (``CREATE`` always persists).
|
|
40
|
+
* **Traversal returns paths, scoring is Python.** AGE has no reliable list
|
|
41
|
+
comprehension (``[r IN ... | ...]``) / ``reduce`` / ``all``. The traversal
|
|
42
|
+
returns ``relationships(path)`` + ``nodes(path)`` as agtype; Python parses
|
|
43
|
+
them, enforces the per-element tenant filter, applies the edge-weight floor,
|
|
44
|
+
and computes ``∏ weights`` affinity (spec §6a).
|
|
45
|
+
"""
|
|
46
|
+
from __future__ import annotations
|
|
47
|
+
|
|
48
|
+
import json
|
|
49
|
+
from collections.abc import Iterator, Mapping, Sequence
|
|
50
|
+
from contextlib import contextmanager
|
|
51
|
+
from typing import Any
|
|
52
|
+
|
|
53
|
+
import psycopg
|
|
54
|
+
from psycopg.pq import TransactionStatus
|
|
55
|
+
|
|
56
|
+
from ...db import DEFAULT_GRAPH_NAME, bootstrap_age
|
|
57
|
+
from ...errors import GraphBackendError
|
|
58
|
+
from ..schema import EntityMention, GraphEntity
|
|
59
|
+
from ._age_helpers import (
|
|
60
|
+
_DOCUMENT_RESERVED_KEYS,
|
|
61
|
+
_EDGE_LABELS,
|
|
62
|
+
_ENTITY_RESERVED_KEYS,
|
|
63
|
+
_GRAPH_NAME_RE,
|
|
64
|
+
_PROPERTY_INDEXES,
|
|
65
|
+
_TRAVERSE_MAX_PATHS_FLOOR,
|
|
66
|
+
_TRAVERSE_MAX_PATHS_MULTIPLIER,
|
|
67
|
+
_VERTEX_LABELS,
|
|
68
|
+
_agtype_loads,
|
|
69
|
+
_all_same_tenant,
|
|
70
|
+
_edge_weight,
|
|
71
|
+
_inline_set_map,
|
|
72
|
+
_property_index_ddl,
|
|
73
|
+
_require_autocommit,
|
|
74
|
+
_require_positive_int,
|
|
75
|
+
_require_weight_floor,
|
|
76
|
+
_scope_person_relational,
|
|
77
|
+
)
|
|
78
|
+
from .base import PersonScope, TraversalHit
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
class AgeBackend:
|
|
82
|
+
"""Apache AGE backend — parameterized Cypher over one tenant-scoped graph.
|
|
83
|
+
|
|
84
|
+
Conforms structurally to :class:`brain.graph_rag.backends.base.GraphBackend`.
|
|
85
|
+
Construct once (cheap; holds only the validated graph name) and pass a
|
|
86
|
+
connection per call. The connection must have AGE loaded
|
|
87
|
+
(:func:`brain.db.connect_age`); :meth:`bootstrap` additionally requires an
|
|
88
|
+
autocommit connection (catalog DDL).
|
|
89
|
+
"""
|
|
90
|
+
|
|
91
|
+
def __init__(self, graph_name: str = DEFAULT_GRAPH_NAME) -> None:
|
|
92
|
+
if not _GRAPH_NAME_RE.match(graph_name):
|
|
93
|
+
raise GraphBackendError(
|
|
94
|
+
f"invalid AGE graph name {graph_name!r}: must be a SQL identifier "
|
|
95
|
+
r"matching ^[A-Za-z_][A-Za-z0-9_]*$"
|
|
96
|
+
)
|
|
97
|
+
self._graph_name = graph_name
|
|
98
|
+
|
|
99
|
+
@property
|
|
100
|
+
def graph_name(self) -> str:
|
|
101
|
+
"""The validated AGE graph this backend operates on."""
|
|
102
|
+
return self._graph_name
|
|
103
|
+
|
|
104
|
+
# ------------------------------------------------------------------ #
|
|
105
|
+
# Session / execution helpers
|
|
106
|
+
# ------------------------------------------------------------------ #
|
|
107
|
+
@contextmanager
|
|
108
|
+
def _age_session(self, conn: psycopg.Connection[Any]) -> Iterator[None]:
|
|
109
|
+
"""Put ``ag_catalog`` on the search_path for the enclosed Cypher.
|
|
110
|
+
|
|
111
|
+
Required so AGE's operators/casts (``=``, ``@>``, agtype→boolean)
|
|
112
|
+
resolve. Restores the default search_path on exit — never a global leak.
|
|
113
|
+
If the transaction aborted inside the block, the RESET is skipped (the
|
|
114
|
+
caller's rollback restores search_path; issuing SQL on an aborted
|
|
115
|
+
transaction would itself error).
|
|
116
|
+
"""
|
|
117
|
+
conn.execute('SET search_path = ag_catalog, "$user", public')
|
|
118
|
+
try:
|
|
119
|
+
yield
|
|
120
|
+
finally:
|
|
121
|
+
if conn.info.transaction_status != TransactionStatus.INERROR:
|
|
122
|
+
conn.execute("RESET search_path")
|
|
123
|
+
|
|
124
|
+
def _cypher(
|
|
125
|
+
self,
|
|
126
|
+
conn: psycopg.Connection[Any],
|
|
127
|
+
query: str,
|
|
128
|
+
params: Mapping[str, Any] | None = None,
|
|
129
|
+
*,
|
|
130
|
+
columns: str = "v ag_catalog.agtype",
|
|
131
|
+
) -> list[tuple[Any, ...]]:
|
|
132
|
+
"""Run one ``ag_catalog.cypher`` statement and return its rows.
|
|
133
|
+
|
|
134
|
+
``query`` is a backend-generated Cypher template (it may reference
|
|
135
|
+
``$key`` params); ``params`` supplies those values and is the *only*
|
|
136
|
+
bound input, sent as ``%s::ag_catalog.agtype`` over a JSON string.
|
|
137
|
+
``columns`` is the generated ``AS (...)`` column list. Any
|
|
138
|
+
``psycopg.Error`` is wrapped in :class:`GraphBackendError`.
|
|
139
|
+
"""
|
|
140
|
+
graph = self._graph_name
|
|
141
|
+
if params is None:
|
|
142
|
+
statement = (
|
|
143
|
+
f"SELECT * FROM ag_catalog.cypher('{graph}', $$ {query} $$) "
|
|
144
|
+
f"AS ({columns})"
|
|
145
|
+
)
|
|
146
|
+
args: tuple[Any, ...] = ()
|
|
147
|
+
else:
|
|
148
|
+
statement = (
|
|
149
|
+
f"SELECT * FROM ag_catalog.cypher('{graph}', $$ {query} $$, "
|
|
150
|
+
f"%s::ag_catalog.agtype) AS ({columns})"
|
|
151
|
+
)
|
|
152
|
+
args = (json.dumps(params),)
|
|
153
|
+
try:
|
|
154
|
+
return conn.execute(statement, args).fetchall()
|
|
155
|
+
except psycopg.Error as exc:
|
|
156
|
+
raise GraphBackendError(
|
|
157
|
+
f"Cypher execution failed on graph {graph!r}: {exc}"
|
|
158
|
+
) from exc
|
|
159
|
+
|
|
160
|
+
def _label_exists(self, conn: psycopg.Connection[Any], label: str) -> bool:
|
|
161
|
+
"""True iff ``label`` already exists in this backend's graph."""
|
|
162
|
+
row = conn.execute(
|
|
163
|
+
"SELECT 1 FROM ag_catalog.ag_label l "
|
|
164
|
+
"JOIN ag_catalog.ag_graph g ON l.graph = g.graphid "
|
|
165
|
+
"WHERE g.name = %s AND l.name = %s",
|
|
166
|
+
(self._graph_name, label),
|
|
167
|
+
).fetchone()
|
|
168
|
+
return row is not None
|
|
169
|
+
|
|
170
|
+
# ------------------------------------------------------------------ #
|
|
171
|
+
# bootstrap
|
|
172
|
+
# ------------------------------------------------------------------ #
|
|
173
|
+
def bootstrap(self, conn: psycopg.Connection[Any]) -> None:
|
|
174
|
+
"""Provision graph + labels + property indexes, idempotently (spec §5b)."""
|
|
175
|
+
_require_autocommit(conn, "AgeBackend.bootstrap")
|
|
176
|
+
# 1. Ensure the graph (+ extension) exists. Reuses the G0-2 bootstrap,
|
|
177
|
+
# which is itself idempotent (creates the graph only when absent).
|
|
178
|
+
bootstrap_age(conn, self._graph_name)
|
|
179
|
+
try:
|
|
180
|
+
# 2. Labels. create_vlabel/create_elabel are NOT idempotent (they
|
|
181
|
+
# raise InvalidSchemaName on a second call), so guard each via the
|
|
182
|
+
# ag_label catalog. Fully ag_catalog-qualified -> no search_path.
|
|
183
|
+
for label in _VERTEX_LABELS:
|
|
184
|
+
if not self._label_exists(conn, label):
|
|
185
|
+
conn.execute(
|
|
186
|
+
"SELECT ag_catalog.create_vlabel(%s, %s)",
|
|
187
|
+
(self._graph_name, label),
|
|
188
|
+
)
|
|
189
|
+
for label in _EDGE_LABELS:
|
|
190
|
+
if not self._label_exists(conn, label):
|
|
191
|
+
conn.execute(
|
|
192
|
+
"SELECT ag_catalog.create_elabel(%s, %s)",
|
|
193
|
+
(self._graph_name, label),
|
|
194
|
+
)
|
|
195
|
+
# 3. Property indexes on the label backing tables. CREATE INDEX IF
|
|
196
|
+
# NOT EXISTS is idempotent; the expression is fully ag_catalog-
|
|
197
|
+
# qualified, so no search_path is required.
|
|
198
|
+
for label, prop, idx_name in _PROPERTY_INDEXES:
|
|
199
|
+
conn.execute(
|
|
200
|
+
_property_index_ddl(self._graph_name, label, prop, idx_name)
|
|
201
|
+
)
|
|
202
|
+
except psycopg.Error as exc:
|
|
203
|
+
raise GraphBackendError(
|
|
204
|
+
f"failed to bootstrap AGE labels/indexes on graph "
|
|
205
|
+
f"{self._graph_name!r}: {exc}"
|
|
206
|
+
) from exc
|
|
207
|
+
|
|
208
|
+
# ------------------------------------------------------------------ #
|
|
209
|
+
# Writes
|
|
210
|
+
# ------------------------------------------------------------------ #
|
|
211
|
+
def upsert_entities(
|
|
212
|
+
self,
|
|
213
|
+
conn: psycopg.Connection[Any],
|
|
214
|
+
tenant_id: str,
|
|
215
|
+
entities: Sequence[GraphEntity],
|
|
216
|
+
) -> int:
|
|
217
|
+
"""MERGE Entity vertices, then MATCH+SET their mutable properties.
|
|
218
|
+
|
|
219
|
+
**Atomic, all-or-nothing** (matches :meth:`refresh_cooccur_edges` /
|
|
220
|
+
:meth:`_detach_delete_vertices`): the whole MERGE+SET batch runs inside one
|
|
221
|
+
``conn.transaction()`` — a real transaction on an autocommit connection, a
|
|
222
|
+
SAVEPOINT when already nested (e.g. under the reconcile transaction or the
|
|
223
|
+
``build --force`` pre-pass) — so a failure partway through rolls the batch
|
|
224
|
+
back rather than leaving some vertices created and others not.
|
|
225
|
+
|
|
226
|
+
**Batched (perf-T4 G2).** A single ``UNWIND $rows`` Cypher MERGEs every
|
|
227
|
+
vertex in one round-trip, then a second ``UNWIND $rows`` Cypher MATCHes
|
|
228
|
+
and SETs each vertex's mutable properties. The two statements are split
|
|
229
|
+
because a ``SET`` in the same statement as a freshly-creating MERGE does
|
|
230
|
+
not persist (AGE quirk — see this module's preamble). Two round-trips
|
|
231
|
+
per call instead of ``2 × N`` (13,190 statements for a 6,595-entity
|
|
232
|
+
build becomes 2).
|
|
233
|
+
"""
|
|
234
|
+
if not entities:
|
|
235
|
+
return 0
|
|
236
|
+
# Validate cross-tenant + build the per-row params upfront, BEFORE
|
|
237
|
+
# opening the AGE session, so a bad input never touches the graph.
|
|
238
|
+
for entity in entities:
|
|
239
|
+
if entity.tenant_id != tenant_id:
|
|
240
|
+
raise GraphBackendError(
|
|
241
|
+
f"cross-tenant entity upsert: entity {entity.id} carries "
|
|
242
|
+
f"tenant_id {entity.tenant_id!r} but the call scopes "
|
|
243
|
+
f"tenant_id {tenant_id!r}"
|
|
244
|
+
)
|
|
245
|
+
# Fixed backend-controlled keys (not a caller bag) — run the reserved-key
|
|
246
|
+
# guard once on a sample so a future edit that added an identity key here
|
|
247
|
+
# would still be caught, matching the per-row form's defence in depth.
|
|
248
|
+
_inline_set_map(
|
|
249
|
+
{"name": "", "entity_type": "", "canonical_key": ""},
|
|
250
|
+
reserved=_ENTITY_RESERVED_KEYS,
|
|
251
|
+
context="entity properties",
|
|
252
|
+
)
|
|
253
|
+
rows = [
|
|
254
|
+
{
|
|
255
|
+
"u": entity.id,
|
|
256
|
+
"name": entity.name,
|
|
257
|
+
"entity_type": entity.entity_type,
|
|
258
|
+
"canonical_key": entity.canonical_key,
|
|
259
|
+
}
|
|
260
|
+
for entity in entities
|
|
261
|
+
]
|
|
262
|
+
with self._age_session(conn), conn.transaction():
|
|
263
|
+
# Pass 1: MERGE every vertex in one Cypher round-trip. AGE's UNWIND
|
|
264
|
+
# iterates the agtype list parameter; the MERGE pattern reads the
|
|
265
|
+
# per-row identity keys via ``row.<key>`` map access.
|
|
266
|
+
self._cypher(
|
|
267
|
+
conn,
|
|
268
|
+
"UNWIND $rows AS row "
|
|
269
|
+
"MERGE (e:Entity {entity_uuid: row.u, tenant_id: $t})",
|
|
270
|
+
{"t": tenant_id, "rows": rows},
|
|
271
|
+
)
|
|
272
|
+
# Pass 2: MATCH each vertex (always on the match path → SET persists)
|
|
273
|
+
# and update its mutable properties. The keys are fixed identifiers
|
|
274
|
+
# interpolated into Cypher; the values flow as agtype params via row.
|
|
275
|
+
self._cypher(
|
|
276
|
+
conn,
|
|
277
|
+
"UNWIND $rows AS row "
|
|
278
|
+
"MATCH (e:Entity {entity_uuid: row.u, tenant_id: $t}) "
|
|
279
|
+
"SET e.name = row.name, "
|
|
280
|
+
"e.entity_type = row.entity_type, "
|
|
281
|
+
"e.canonical_key = row.canonical_key",
|
|
282
|
+
{"t": tenant_id, "rows": rows},
|
|
283
|
+
)
|
|
284
|
+
return len(entities)
|
|
285
|
+
|
|
286
|
+
def upsert_mention_edges(
|
|
287
|
+
self,
|
|
288
|
+
conn: psycopg.Connection[Any],
|
|
289
|
+
tenant_id: str,
|
|
290
|
+
document_id: str,
|
|
291
|
+
mentions: Sequence[EntityMention],
|
|
292
|
+
*,
|
|
293
|
+
document_props: Mapping[str, Any] | None = None,
|
|
294
|
+
) -> int:
|
|
295
|
+
"""Split-MERGE the doc + entities, then delete+recreate MENTIONED_IN.
|
|
296
|
+
|
|
297
|
+
**Atomic, all-or-nothing** (matches :meth:`upsert_entities` /
|
|
298
|
+
:meth:`refresh_cooccur_edges` / :meth:`_detach_delete_vertices`): the
|
|
299
|
+
whole MERGE + delete + recreate batch runs inside one
|
|
300
|
+
``conn.transaction()`` — a real transaction on an autocommit connection, a
|
|
301
|
+
SAVEPOINT when already nested (e.g. under the reconcile transaction). The
|
|
302
|
+
step-3 DELETE drops ALL of this document's existing MENTIONED_IN edges
|
|
303
|
+
before step 4 recreates them, so without the transaction a failure partway
|
|
304
|
+
through the recreate loop would leave a half-rebuilt edge set (the DELETE
|
|
305
|
+
already committed, only some CREATEs applied) for any caller outside
|
|
306
|
+
reconcile's own transaction. Wrapping here makes it either fully replaced
|
|
307
|
+
or left exactly as it was.
|
|
308
|
+
"""
|
|
309
|
+
with self._age_session(conn), conn.transaction():
|
|
310
|
+
# 1. MERGE the Document vertex (split MERGE — vertex first).
|
|
311
|
+
self._cypher(
|
|
312
|
+
conn,
|
|
313
|
+
"MERGE (d:Document {document_uuid: $d, tenant_id: $t})",
|
|
314
|
+
{"d": document_id, "t": tenant_id},
|
|
315
|
+
)
|
|
316
|
+
if document_props:
|
|
317
|
+
# Caller-supplied bag — reject any attempt to overwrite the
|
|
318
|
+
# Document's identity keys before building the SET map.
|
|
319
|
+
map_clause, map_params = _inline_set_map(
|
|
320
|
+
document_props,
|
|
321
|
+
reserved=_DOCUMENT_RESERVED_KEYS,
|
|
322
|
+
context="document_props",
|
|
323
|
+
)
|
|
324
|
+
self._cypher(
|
|
325
|
+
conn,
|
|
326
|
+
"MATCH (d:Document {document_uuid: $d, tenant_id: $t}) "
|
|
327
|
+
f"SET d += {map_clause}",
|
|
328
|
+
{"d": document_id, "t": tenant_id, **map_params},
|
|
329
|
+
)
|
|
330
|
+
# 2. MERGE each Entity vertex (defensive get-or-create; properties
|
|
331
|
+
# are owned by upsert_entities, so no SET here).
|
|
332
|
+
for mention in mentions:
|
|
333
|
+
if mention.tenant_id != tenant_id or mention.document_id != document_id:
|
|
334
|
+
raise GraphBackendError(
|
|
335
|
+
"mention does not match the call scope: "
|
|
336
|
+
f"got tenant_id={mention.tenant_id!r} "
|
|
337
|
+
f"document_id={mention.document_id!r}, expected "
|
|
338
|
+
f"tenant_id={tenant_id!r} document_id={document_id!r}"
|
|
339
|
+
)
|
|
340
|
+
self._cypher(
|
|
341
|
+
conn,
|
|
342
|
+
"MERGE (e:Entity {entity_uuid: $e, tenant_id: $t})",
|
|
343
|
+
{"e": mention.entity_id, "t": tenant_id},
|
|
344
|
+
)
|
|
345
|
+
# 3. Delete this document's existing MENTIONED_IN edges (recreate
|
|
346
|
+
# below) — keeps re-ingest idempotent (spec §7.3).
|
|
347
|
+
self._cypher(
|
|
348
|
+
conn,
|
|
349
|
+
"MATCH (:Entity)-[r:MENTIONED_IN {tenant_id: $t}]->"
|
|
350
|
+
"(d:Document {document_uuid: $d, tenant_id: $t}) DELETE r",
|
|
351
|
+
{"d": document_id, "t": tenant_id},
|
|
352
|
+
)
|
|
353
|
+
# 4. Recreate the edges with inline properties (CREATE persists).
|
|
354
|
+
for mention in mentions:
|
|
355
|
+
self._cypher(
|
|
356
|
+
conn,
|
|
357
|
+
"MATCH (e:Entity {entity_uuid: $e, tenant_id: $t}) "
|
|
358
|
+
"MATCH (d:Document {document_uuid: $d, tenant_id: $t}) "
|
|
359
|
+
"CREATE (e)-[:MENTIONED_IN "
|
|
360
|
+
"{tenant_id: $t, mention_count: $c, source: $s}]->(d)",
|
|
361
|
+
{
|
|
362
|
+
"e": mention.entity_id,
|
|
363
|
+
"d": document_id,
|
|
364
|
+
"t": tenant_id,
|
|
365
|
+
"c": mention.mention_count,
|
|
366
|
+
"s": mention.source,
|
|
367
|
+
},
|
|
368
|
+
)
|
|
369
|
+
return len(mentions)
|
|
370
|
+
|
|
371
|
+
def refresh_cooccur_edges(
|
|
372
|
+
self,
|
|
373
|
+
conn: psycopg.Connection[Any],
|
|
374
|
+
tenant_id: str,
|
|
375
|
+
) -> int:
|
|
376
|
+
"""Rematerialize CO_OCCURS from the relational ``graph_relationships`` mirror.
|
|
377
|
+
|
|
378
|
+
Reads the (already weighted) aggregate rows for the tenant, then in AGE
|
|
379
|
+
deletes the tenant's CO_OCCURS edges and recreates them. Computing the
|
|
380
|
+
normalized-lift weight from contributions is upstream (G1 weighting);
|
|
381
|
+
this primitive mirrors the relational aggregate into the graph.
|
|
382
|
+
|
|
383
|
+
**Atomic, complete-or-loud-failure contract** (matches :meth:`traverse`
|
|
384
|
+
/ :meth:`scope_person`): the returned count is the number of edges
|
|
385
|
+
ACTUALLY created, never an optimistic ``len(rows)``. Each recreate is a
|
|
386
|
+
``MATCH (a) MATCH (b) CREATE ... RETURN`` — if either endpoint Entity
|
|
387
|
+
vertex is missing, both MATCHes fail to bind, the CREATE silently
|
|
388
|
+
produces nothing, and ``RETURN`` yields zero rows. We DETECT that empty
|
|
389
|
+
result and raise :class:`GraphBackendError` rather than over-reporting a
|
|
390
|
+
write that didn't happen. (The relational ``graph_relationships`` mirror
|
|
391
|
+
carries tenant-safe FKs to ``graph_entities`` rows, but the AGE *vertex*
|
|
392
|
+
is provisioned separately by :meth:`upsert_entities`; a missing vertex
|
|
393
|
+
means the caller refreshed edges before upserting entities — a bug worth
|
|
394
|
+
surfacing, not swallowing.)
|
|
395
|
+
|
|
396
|
+
The delete + all recreates run inside ONE explicit transaction
|
|
397
|
+
(``conn.transaction()`` — a real transaction under an autocommit
|
|
398
|
+
connection, a SAVEPOINT when already inside one). A detected miss (or any
|
|
399
|
+
other failure mid-rebuild) raises, which rolls the whole block back: the
|
|
400
|
+
tenant's CO_OCCURS set is either FULLY replaced or left exactly as it
|
|
401
|
+
was — never the partial state a bare delete-then-create would leave
|
|
402
|
+
(old edges already dropped, only some recreated) on an autocommit
|
|
403
|
+
connection.
|
|
404
|
+
"""
|
|
405
|
+
# Read the relational mirror first, under the default search_path so the
|
|
406
|
+
# public table resolves cleanly.
|
|
407
|
+
rows = conn.execute(
|
|
408
|
+
"SELECT src_id, dst_id, weight, co_count, doc_count, rel_type "
|
|
409
|
+
"FROM graph_relationships WHERE tenant_id = %s",
|
|
410
|
+
(tenant_id,),
|
|
411
|
+
).fetchall()
|
|
412
|
+
with self._age_session(conn), conn.transaction():
|
|
413
|
+
# Full recompute, ATOMIC: drop the tenant's aggregate edges, then
|
|
414
|
+
# recreate — all inside one transaction so a miss below rolls the
|
|
415
|
+
# delete back too (no partial replacement).
|
|
416
|
+
self._cypher(
|
|
417
|
+
conn,
|
|
418
|
+
"MATCH ()-[r:CO_OCCURS {tenant_id: $t}]->() DELETE r",
|
|
419
|
+
{"t": tenant_id},
|
|
420
|
+
)
|
|
421
|
+
if not rows:
|
|
422
|
+
return 0
|
|
423
|
+
# Batched (perf-T4 G1): one UNWIND Cypher per call instead of N
|
|
424
|
+
# round-trips. AGE iterates the agtype list parameter; each row's
|
|
425
|
+
# endpoint UUIDs / edge properties flow through ``row.<key>`` map
|
|
426
|
+
# access. ``RETURN row.s, row.d`` emits one result row per (a,b)
|
|
427
|
+
# pair where BOTH MATCHes bound and the CREATE ran — when either
|
|
428
|
+
# MATCH misses, the UNWIND row simply produces nothing. Comparing
|
|
429
|
+
# the returned (src,dst) set against the input set lets us preserve
|
|
430
|
+
# the original complete-or-loud-failure miss-detection contract.
|
|
431
|
+
edge_rows = [
|
|
432
|
+
{
|
|
433
|
+
"s": str(src_id),
|
|
434
|
+
"d": str(dst_id),
|
|
435
|
+
"w": float(weight),
|
|
436
|
+
"co": int(co_count),
|
|
437
|
+
"dc": int(doc_count),
|
|
438
|
+
"rt": rel_type,
|
|
439
|
+
}
|
|
440
|
+
for src_id, dst_id, weight, co_count, doc_count, rel_type in rows
|
|
441
|
+
]
|
|
442
|
+
result = self._cypher(
|
|
443
|
+
conn,
|
|
444
|
+
"UNWIND $rows AS row "
|
|
445
|
+
"MATCH (a:Entity {entity_uuid: row.s, tenant_id: $t}) "
|
|
446
|
+
"MATCH (b:Entity {entity_uuid: row.d, tenant_id: $t}) "
|
|
447
|
+
"CREATE (a)-[:CO_OCCURS {tenant_id: $t, weight: row.w, "
|
|
448
|
+
"co_count: row.co, doc_count: row.dc, rel_type: row.rt}]->(b) "
|
|
449
|
+
"RETURN row.s, row.d",
|
|
450
|
+
{"t": tenant_id, "rows": edge_rows},
|
|
451
|
+
columns="s ag_catalog.agtype, d ag_catalog.agtype",
|
|
452
|
+
)
|
|
453
|
+
created = len(result)
|
|
454
|
+
if created < len(edge_rows):
|
|
455
|
+
# At least one MATCH missed → the relational mirror references
|
|
456
|
+
# an entity with no AGE vertex. Identify the FIRST missing pair
|
|
457
|
+
# for a precise message (matches the prior per-row form's error
|
|
458
|
+
# shape), then raise. Raising aborts conn.transaction() → the
|
|
459
|
+
# DELETE and any CREATEs in this rebuild roll back, so the prior
|
|
460
|
+
# CO_OCCURS set is preserved intact (all-or-nothing).
|
|
461
|
+
returned = {(_agtype_loads(s), _agtype_loads(d)) for s, d in result}
|
|
462
|
+
first_missing = next(
|
|
463
|
+
((r["s"], r["d"]) for r in edge_rows
|
|
464
|
+
if (r["s"], r["d"]) not in returned),
|
|
465
|
+
(None, None),
|
|
466
|
+
)
|
|
467
|
+
raise GraphBackendError(
|
|
468
|
+
"refresh_cooccur_edges could not create a CO_OCCURS edge "
|
|
469
|
+
f"for tenant {tenant_id!r}: an endpoint Entity vertex is "
|
|
470
|
+
f"missing (src={first_missing[0]} dst={first_missing[1]}). "
|
|
471
|
+
"The relational graph_relationships mirror references an "
|
|
472
|
+
"entity with no AGE vertex — run upsert_entities before "
|
|
473
|
+
"refreshing edges."
|
|
474
|
+
)
|
|
475
|
+
return created
|
|
476
|
+
|
|
477
|
+
# ------------------------------------------------------------------ #
|
|
478
|
+
# Reads / traversal (the §6a canary)
|
|
479
|
+
# ------------------------------------------------------------------ #
|
|
480
|
+
def traverse(
|
|
481
|
+
self,
|
|
482
|
+
conn: psycopg.Connection[Any],
|
|
483
|
+
tenant_id: str,
|
|
484
|
+
seed_entity_uuid: str,
|
|
485
|
+
*,
|
|
486
|
+
depth: int,
|
|
487
|
+
frontier_cap: int,
|
|
488
|
+
min_edge_weight: float = 0.0,
|
|
489
|
+
) -> list[TraversalHit]:
|
|
490
|
+
"""Bounded CO_OCCURS traversal; Python scores affinity (spec §6a).
|
|
491
|
+
|
|
492
|
+
The seed is never returned (excluded in Cypher AND skipped in Python).
|
|
493
|
+
The functional ``frontier_cap`` is applied in Python *after* affinity
|
|
494
|
+
scoring + dedupe + the tenant re-check, so the highest-affinity entities
|
|
495
|
+
are returned deterministically — a Cypher ``LIMIT`` before scoring would
|
|
496
|
+
drop arbitrary (possibly best) paths since AGE cannot order by affinity
|
|
497
|
+
(no reduce/list-comprehension). Ties on affinity break by ``entity_uuid``
|
|
498
|
+
ascending, so the top-``frontier_cap`` selection is fully reproducible.
|
|
499
|
+
|
|
500
|
+
**Correctness-or-failure contract:** traverse() scores EVERY within-depth
|
|
501
|
+
path. To stay bounded it fetches at most ``max(frontier_cap *
|
|
502
|
+
_TRAVERSE_MAX_PATHS_MULTIPLIER, _TRAVERSE_MAX_PATHS_FLOOR)`` paths and
|
|
503
|
+
*detects* overflow (it asks AGE for one extra row); if more paths exist
|
|
504
|
+
than the bound it raises :class:`GraphBackendError` rather than returning
|
|
505
|
+
a silently-truncated, possibly-wrong best-per-entity. So a result is
|
|
506
|
+
either complete/correct or a loud failure. Tightening the bound / perf at
|
|
507
|
+
scale is deferred to G2's P95 gate.
|
|
508
|
+
|
|
509
|
+
Tenant safety: every path edge is filtered inline (``{tenant_id: $t}``)
|
|
510
|
+
and both queried endpoints are constrained in the ``WHERE``. AGE has no
|
|
511
|
+
``all()``/list-comprehension to constrain *intermediate* nodes in-query,
|
|
512
|
+
so a Python per-element re-check over ``nodes(path)``/``relationships
|
|
513
|
+
(path)`` drops any path touching a foreign element BEFORE the cap is
|
|
514
|
+
applied — no foreign vertex/edge is ever returned. (Relational
|
|
515
|
+
composite-FK integrity already prevents a same-tenant edge from pointing
|
|
516
|
+
at a foreign vertex, so the re-check is defence in depth, not the sole
|
|
517
|
+
guard.)
|
|
518
|
+
"""
|
|
519
|
+
_require_positive_int(depth, "depth")
|
|
520
|
+
_require_positive_int(frontier_cap, "frontier_cap")
|
|
521
|
+
_require_weight_floor(min_edge_weight)
|
|
522
|
+
# ``depth`` is baked into the fixed ``*1..N`` template (a variable-length
|
|
523
|
+
# pattern cannot use a bound depth). ``max_paths`` is the correctness
|
|
524
|
+
# bound: we fetch one EXTRA row (``LIMIT max_paths + 1``) so that >
|
|
525
|
+
# max_paths rows means the result would be incomplete → raise instead of
|
|
526
|
+
# silently truncating. Both values are validated ints.
|
|
527
|
+
max_paths = max(
|
|
528
|
+
frontier_cap * _TRAVERSE_MAX_PATHS_MULTIPLIER,
|
|
529
|
+
_TRAVERSE_MAX_PATHS_FLOOR,
|
|
530
|
+
)
|
|
531
|
+
fetch_limit = max_paths + 1
|
|
532
|
+
query = (
|
|
533
|
+
f"MATCH path = (s:Entity)-[:CO_OCCURS*1..{depth} "
|
|
534
|
+
"{tenant_id: $t}]-(n:Entity) "
|
|
535
|
+
"WHERE s.entity_uuid = $seed AND s.tenant_id = $t AND n.tenant_id = $t "
|
|
536
|
+
"AND n.entity_uuid <> $seed "
|
|
537
|
+
"RETURN n.entity_uuid AS eid, relationships(path) AS rels, "
|
|
538
|
+
f"nodes(path) AS nds, length(path) AS hops LIMIT {fetch_limit}"
|
|
539
|
+
)
|
|
540
|
+
with self._age_session(conn):
|
|
541
|
+
rows = self._cypher(
|
|
542
|
+
conn,
|
|
543
|
+
query,
|
|
544
|
+
{"seed": seed_entity_uuid, "t": tenant_id},
|
|
545
|
+
columns=(
|
|
546
|
+
"eid ag_catalog.agtype, rels ag_catalog.agtype, "
|
|
547
|
+
"nds ag_catalog.agtype, hops ag_catalog.agtype"
|
|
548
|
+
),
|
|
549
|
+
)
|
|
550
|
+
if len(rows) > max_paths:
|
|
551
|
+
# More within-depth paths than we can safely score → returning now
|
|
552
|
+
# would be a silently-truncated (possibly wrong) best-per-entity.
|
|
553
|
+
# Fail loudly instead (perf redesign is G2's P95 gate).
|
|
554
|
+
raise GraphBackendError(
|
|
555
|
+
f"traversal exceeded safe path bound ({max_paths}) at "
|
|
556
|
+
f"depth={depth} from seed {seed_entity_uuid!r}; narrow the scope "
|
|
557
|
+
"(lower depth) — perf redesign for large frontiers is deferred to "
|
|
558
|
+
"G2"
|
|
559
|
+
)
|
|
560
|
+
|
|
561
|
+
# Score every fetched path in Python, then dedupe to the best per entity,
|
|
562
|
+
# sort by affinity (ties broken by entity_uuid for determinism), and ONLY
|
|
563
|
+
# THEN apply the functional frontier cap.
|
|
564
|
+
best: dict[str, TraversalHit] = {}
|
|
565
|
+
for eid_raw, rels_raw, nds_raw, hops_raw in rows:
|
|
566
|
+
eid = _agtype_loads(eid_raw)
|
|
567
|
+
# Belt + suspenders: never return the seed even if a cycle slips past
|
|
568
|
+
# the Cypher ``n.entity_uuid <> $seed`` guard.
|
|
569
|
+
if eid == seed_entity_uuid:
|
|
570
|
+
continue
|
|
571
|
+
rels = _agtype_loads(rels_raw) or []
|
|
572
|
+
nodes = _agtype_loads(nds_raw) or []
|
|
573
|
+
hops = int(_agtype_loads(hops_raw))
|
|
574
|
+
# Per-element tenant re-check (incl. intermediate nodes) BEFORE the
|
|
575
|
+
# cap, so a foreign path can never consume frontier budget.
|
|
576
|
+
if not _all_same_tenant(rels, nodes, tenant_id):
|
|
577
|
+
continue
|
|
578
|
+
weights = [_edge_weight(edge) for edge in rels]
|
|
579
|
+
if any(w < min_edge_weight for w in weights):
|
|
580
|
+
continue
|
|
581
|
+
affinity = 1.0
|
|
582
|
+
for w in weights:
|
|
583
|
+
affinity *= w
|
|
584
|
+
current = best.get(eid)
|
|
585
|
+
if current is None or affinity > current.affinity:
|
|
586
|
+
best[eid] = TraversalHit(
|
|
587
|
+
entity_uuid=eid,
|
|
588
|
+
affinity=affinity,
|
|
589
|
+
hops=hops,
|
|
590
|
+
tenant_id=tenant_id,
|
|
591
|
+
)
|
|
592
|
+
# Affinity DESC, ties broken by entity_uuid ASC → reproducible ordering
|
|
593
|
+
# (and therefore a reproducible top-frontier_cap selection).
|
|
594
|
+
hits = sorted(best.values(), key=lambda h: (-h.affinity, h.entity_uuid))
|
|
595
|
+
return hits[:frontier_cap]
|
|
596
|
+
|
|
597
|
+
def scope_person(
|
|
598
|
+
self,
|
|
599
|
+
conn: psycopg.Connection[Any],
|
|
600
|
+
tenant_id: str,
|
|
601
|
+
seed_entity_uuid: str,
|
|
602
|
+
*,
|
|
603
|
+
frontier_cap: int,
|
|
604
|
+
) -> PersonScope:
|
|
605
|
+
"""Person -> docs -> co-mentioned entities, tenant-scoped (spec §6b).
|
|
606
|
+
|
|
607
|
+
Delegates to the instance-state-free
|
|
608
|
+
:func:`brain.graph_rag.backends._age_helpers._scope_person_relational`
|
|
609
|
+
(an indexed RELATIONAL two-hop over ``graph_entity_mentions`` — NOT
|
|
610
|
+
Cypher — see there for the full rationale + the complete-or-loud-failure
|
|
611
|
+
overflow contract). Kept as a method so the backend satisfies the
|
|
612
|
+
:class:`~brain.graph_rag.backends.base.GraphBackend` Protocol.
|
|
613
|
+
"""
|
|
614
|
+
return _scope_person_relational(
|
|
615
|
+
conn, tenant_id, seed_entity_uuid, frontier_cap=frontier_cap
|
|
616
|
+
)
|
|
617
|
+
|
|
618
|
+
# ------------------------------------------------------------------ #
|
|
619
|
+
# Targeted vertex GC (reconcile orphan cleanup — spec §7)
|
|
620
|
+
# ------------------------------------------------------------------ #
|
|
621
|
+
def detach_delete_entities(
|
|
622
|
+
self,
|
|
623
|
+
conn: psycopg.Connection[Any],
|
|
624
|
+
tenant_id: str,
|
|
625
|
+
entity_uuids: Sequence[str],
|
|
626
|
+
) -> int:
|
|
627
|
+
"""DETACH DELETE specific Entity vertices for a tenant (orphan GC)."""
|
|
628
|
+
return self._detach_delete_vertices(
|
|
629
|
+
conn, "Entity", "entity_uuid", tenant_id, entity_uuids
|
|
630
|
+
)
|
|
631
|
+
|
|
632
|
+
def detach_delete_documents(
|
|
633
|
+
self,
|
|
634
|
+
conn: psycopg.Connection[Any],
|
|
635
|
+
tenant_id: str,
|
|
636
|
+
document_uuids: Sequence[str],
|
|
637
|
+
) -> int:
|
|
638
|
+
"""DETACH DELETE specific Document vertices for a tenant (+ their edges)."""
|
|
639
|
+
return self._detach_delete_vertices(
|
|
640
|
+
conn, "Document", "document_uuid", tenant_id, document_uuids
|
|
641
|
+
)
|
|
642
|
+
|
|
643
|
+
def _detach_delete_vertices(
|
|
644
|
+
self,
|
|
645
|
+
conn: psycopg.Connection[Any],
|
|
646
|
+
label: str,
|
|
647
|
+
id_prop: str,
|
|
648
|
+
tenant_id: str,
|
|
649
|
+
uuids: Sequence[str],
|
|
650
|
+
) -> int:
|
|
651
|
+
"""Count-then-``DETACH DELETE`` each listed vertex of ``label``.
|
|
652
|
+
|
|
653
|
+
``label`` / ``id_prop`` are backend constants (``Entity``/``entity_uuid``
|
|
654
|
+
or ``Document``/``document_uuid``), embedded in the generated Cypher; the
|
|
655
|
+
uuid + tenant are bound agtype params. Each vertex is counted first so
|
|
656
|
+
the returned total reflects vertices that ACTUALLY existed (a uuid with
|
|
657
|
+
no matching vertex contributes zero) — AGE's ``DELETE`` returns no rows
|
|
658
|
+
to count, so a preceding ``RETURN count(v)`` is the reliable signal.
|
|
659
|
+
An empty ``uuids`` sequence is a no-op.
|
|
660
|
+
|
|
661
|
+
**Atomic, all-or-nothing** (matches :meth:`refresh_cooccur_edges`): the
|
|
662
|
+
whole count/delete loop runs inside one ``conn.transaction()`` — a real
|
|
663
|
+
transaction on an autocommit connection, a SAVEPOINT when already nested
|
|
664
|
+
inside the reconcile transaction. A failure partway through rolls the
|
|
665
|
+
block back, so a direct caller can never observe a partial deletion.
|
|
666
|
+
"""
|
|
667
|
+
if not uuids:
|
|
668
|
+
return 0
|
|
669
|
+
deleted = 0
|
|
670
|
+
with self._age_session(conn), conn.transaction():
|
|
671
|
+
for vertex_uuid in uuids:
|
|
672
|
+
rows = self._cypher(
|
|
673
|
+
conn,
|
|
674
|
+
f"MATCH (v:{label} {{{id_prop}: $u, tenant_id: $t}}) "
|
|
675
|
+
"RETURN count(v)",
|
|
676
|
+
{"u": vertex_uuid, "t": tenant_id},
|
|
677
|
+
columns="c ag_catalog.agtype",
|
|
678
|
+
)
|
|
679
|
+
present = int(_agtype_loads(rows[0][0])) if rows else 0
|
|
680
|
+
if present:
|
|
681
|
+
self._cypher(
|
|
682
|
+
conn,
|
|
683
|
+
f"MATCH (v:{label} {{{id_prop}: $u, tenant_id: $t}}) "
|
|
684
|
+
"DETACH DELETE v",
|
|
685
|
+
{"u": vertex_uuid, "t": tenant_id},
|
|
686
|
+
)
|
|
687
|
+
deleted += present
|
|
688
|
+
return deleted
|
|
689
|
+
|
|
690
|
+
# ------------------------------------------------------------------ #
|
|
691
|
+
# Teardown
|
|
692
|
+
# ------------------------------------------------------------------ #
|
|
693
|
+
def clear_tenant(self, conn: psycopg.Connection[Any], tenant_id: str) -> int:
|
|
694
|
+
"""Atomically DETACH DELETE all of one tenant's Entity + Document vertices.
|
|
695
|
+
|
|
696
|
+
The clear-then-rebuild primitive (spec §7) ``build --force`` runs FIRST,
|
|
697
|
+
so the AGE mirror is rebuilt fresh from the relational source-of-truth
|
|
698
|
+
with no stale survivors — a ``Document`` vertex for a doc dropped from the
|
|
699
|
+
relational source, an AGE-only ``Entity`` with no catalog row, and their
|
|
700
|
+
``MENTIONED_IN`` / ``CO_OCCURS`` edges are all removed. Only vertices
|
|
701
|
+
carrying ``tenant_id`` are touched (tenant-safe even with shared
|
|
702
|
+
``entity_uuid`` / ``document_uuid`` values).
|
|
703
|
+
|
|
704
|
+
**Atomic, all-or-nothing** (matches :meth:`refresh_cooccur_edges` /
|
|
705
|
+
:meth:`_detach_delete_vertices`): the count + both DETACH DELETEs run
|
|
706
|
+
inside one ``conn.transaction()`` — a real transaction on an autocommit
|
|
707
|
+
connection, a SAVEPOINT when nested — so a failure between the two
|
|
708
|
+
deletes rolls the whole clear back rather than leaving the Entity set
|
|
709
|
+
gone but the Document set intact. Returns the number of vertices deleted.
|
|
710
|
+
"""
|
|
711
|
+
with self._age_session(conn), conn.transaction():
|
|
712
|
+
deleted = self._count_vertices(conn, "Entity", tenant_id) + (
|
|
713
|
+
self._count_vertices(conn, "Document", tenant_id)
|
|
714
|
+
)
|
|
715
|
+
# DETACH DELETE removes attached MENTIONED_IN / CO_OCCURS edges too.
|
|
716
|
+
self._cypher(
|
|
717
|
+
conn,
|
|
718
|
+
"MATCH (e:Entity {tenant_id: $t}) DETACH DELETE e",
|
|
719
|
+
{"t": tenant_id},
|
|
720
|
+
)
|
|
721
|
+
self._cypher(
|
|
722
|
+
conn,
|
|
723
|
+
"MATCH (d:Document {tenant_id: $t}) DETACH DELETE d",
|
|
724
|
+
{"t": tenant_id},
|
|
725
|
+
)
|
|
726
|
+
return deleted
|
|
727
|
+
|
|
728
|
+
def drop_graph(self, conn: psycopg.Connection[Any], tenant_id: str) -> int:
|
|
729
|
+
"""Tenant-aware teardown — DETACH DELETE the tenant's vertices.
|
|
730
|
+
|
|
731
|
+
Delegates to :meth:`clear_tenant` (teardown and clear-before-rebuild are
|
|
732
|
+
the same operation): kept as a distinct, intention-revealing name for
|
|
733
|
+
decommission callers, and inherits ``clear_tenant``'s atomicity.
|
|
734
|
+
"""
|
|
735
|
+
return self.clear_tenant(conn, tenant_id)
|
|
736
|
+
|
|
737
|
+
def count_entities(self, conn: psycopg.Connection[Any], tenant_id: str) -> int:
|
|
738
|
+
"""Count the tenant's ``Entity`` vertices (doctor drift check; spec §7).
|
|
739
|
+
|
|
740
|
+
Read-only public primitive the ``brain doctor`` graph-drift check uses to
|
|
741
|
+
compare the AGE mirror against the relational ``graph_entities`` count.
|
|
742
|
+
Opens its own :meth:`_age_session` (the caller need not). Any
|
|
743
|
+
``psycopg.Error`` surfaces as :class:`GraphBackendError`.
|
|
744
|
+
"""
|
|
745
|
+
with self._age_session(conn):
|
|
746
|
+
return self._count_vertices(conn, "Entity", tenant_id)
|
|
747
|
+
|
|
748
|
+
def count_cooccur_edges(
|
|
749
|
+
self, conn: psycopg.Connection[Any], tenant_id: str
|
|
750
|
+
) -> int:
|
|
751
|
+
"""Count the tenant's ``CO_OCCURS`` edges (doctor drift check; spec §7).
|
|
752
|
+
|
|
753
|
+
Read-only public primitive the ``brain doctor`` graph-drift check uses to
|
|
754
|
+
compare the AGE mirror against the relational ``graph_relationships``
|
|
755
|
+
count. Opens its own :meth:`_age_session`. Any ``psycopg.Error`` surfaces
|
|
756
|
+
as :class:`GraphBackendError`.
|
|
757
|
+
"""
|
|
758
|
+
with self._age_session(conn):
|
|
759
|
+
rows = self._cypher(
|
|
760
|
+
conn,
|
|
761
|
+
"MATCH ()-[r:CO_OCCURS {tenant_id: $t}]->() RETURN count(r)",
|
|
762
|
+
{"t": tenant_id},
|
|
763
|
+
columns="c ag_catalog.agtype",
|
|
764
|
+
)
|
|
765
|
+
return int(_agtype_loads(rows[0][0])) if rows else 0
|
|
766
|
+
|
|
767
|
+
def _count_vertices(
|
|
768
|
+
self, conn: psycopg.Connection[Any], label: str, tenant_id: str
|
|
769
|
+
) -> int:
|
|
770
|
+
"""Count vertices of a fixed ``label`` carrying ``tenant_id``.
|
|
771
|
+
|
|
772
|
+
``label`` is a backend constant (``Entity``/``Document``), embedded in
|
|
773
|
+
the generated Cypher; only ``tenant_id`` is a bound parameter. Must run
|
|
774
|
+
inside an :meth:`_age_session`.
|
|
775
|
+
"""
|
|
776
|
+
rows = self._cypher(
|
|
777
|
+
conn,
|
|
778
|
+
f"MATCH (v:{label} {{tenant_id: $t}}) RETURN count(v)",
|
|
779
|
+
{"t": tenant_id},
|
|
780
|
+
columns="c ag_catalog.agtype",
|
|
781
|
+
)
|
|
782
|
+
return int(_agtype_loads(rows[0][0])) if rows else 0
|