memuron 0.1.1__py3-none-any.whl

memuron/graphfs/projection.py

@@ -0,0 +1,578 @@
+"""Ledger-fed read model for fast graph filesystem queries."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from artha_engine.store.projection_sql import (
+    sql_store_execute,
+    sql_store_fetchall,
+    sql_store_has_tables,
+)
+
+
+def _json(value: object, default: object) -> object:
+    if value is None:
+        return default
+    if isinstance(value, (dict, list)):
+        return value
+    try:
+        return json.loads(str(value))
+    except json.JSONDecodeError:
+        return default
+
+
+def _dump(value: object) -> str:
+    return json.dumps(value, ensure_ascii=True)
+
+
+def _scope_parts(scope: object) -> tuple[str, str]:
+    values = _json(scope, [])
+    tokens = [str(item) for item in values] if isinstance(values, list) else []
+    org_token = next((token for token in tokens if token.startswith("org:")), "")
+    space_token = next((token for token in tokens if token.startswith("space.")), "")
+    return org_token, space_token
+
+
+def _display(node_type: str, content: str, payload: dict[str, Any]) -> str:
+    if node_type == "collection" and payload.get("name"):
+        return str(payload["name"])
+    if node_type in {"document", "image"}:
+        for key in ("file_name", "filename"):
+            if payload.get(key):
+                return str(payload[key])
+    compact = " ".join(content.split())
+    return compact[:77] + ("..." if len(compact) > 77 else "")
+
+
+class FilesystemProjection:
+    name = "memuron_fs"
+
+    def init(self, store: Any) -> None:
+        if not sql_store_has_tables(store):
+            store.memuron_fs_nodes = getattr(store, "memuron_fs_nodes", {})
+            store.memuron_fs_entries = getattr(store, "memuron_fs_entries", {})
+            store.memuron_fs_edges = getattr(store, "memuron_fs_edges", {})
+            return
+        sql_store_execute(
+            store,
+            """
+            CREATE TABLE IF NOT EXISTS memuron_fs_nodes (
+              node_id TEXT PRIMARY KEY,
+              org_token TEXT NOT NULL DEFAULT '',
+              space_token TEXT NOT NULL DEFAULT '',
+              node_type TEXT NOT NULL DEFAULT 'text',
+              display TEXT NOT NULL,
+              content TEXT NOT NULL,
+              encoding TEXT NOT NULL DEFAULT 'memory',
+              payload_json TEXT NOT NULL DEFAULT '{}',
+              metadata_json TEXT NOT NULL DEFAULT '{}',
+              scope_json TEXT NOT NULL DEFAULT '[]',
+              sequence BIGINT NOT NULL DEFAULT 0
+            )
+            """,
+        )
+        sql_store_execute(
+            store,
+            """
+            CREATE TABLE IF NOT EXISTS memuron_fs_entries (
+              placement_id TEXT PRIMARY KEY,
+              parent_id TEXT NOT NULL,
+              child_id TEXT NOT NULL,
+              name TEXT NOT NULL,
+              org_token TEXT NOT NULL DEFAULT '',
+              space_token TEXT NOT NULL DEFAULT '',
+              inherit_parent_scope BOOLEAN NOT NULL DEFAULT TRUE,
+              metadata_json TEXT NOT NULL DEFAULT '{}',
+              sequence BIGINT NOT NULL DEFAULT 0
+            )
+            """,
+        )
+        sql_store_execute(
+            store,
+            """
+            CREATE TABLE IF NOT EXISTS memuron_fs_edges (
+              edge_id TEXT PRIMARY KEY,
+              source_id TEXT NOT NULL,
+              target_id TEXT NOT NULL,
+              edge_type TEXT NOT NULL,
+              description TEXT NOT NULL DEFAULT '',
+              org_token TEXT NOT NULL DEFAULT '',
+              space_token TEXT NOT NULL DEFAULT '',
+              sequence BIGINT NOT NULL DEFAULT 0
+            )
+            """,
+        )
+        for statement in (
+            "CREATE INDEX IF NOT EXISTS idx_memuron_fs_nodes_scope ON memuron_fs_nodes(org_token, space_token)",
+            "CREATE INDEX IF NOT EXISTS idx_memuron_fs_nodes_scope_type ON memuron_fs_nodes(org_token, space_token, node_type)",
+            "CREATE INDEX IF NOT EXISTS idx_memuron_fs_nodes_encoding ON memuron_fs_nodes(org_token, space_token, encoding)",
+            "CREATE INDEX IF NOT EXISTS idx_memuron_fs_entries_parent ON memuron_fs_entries(parent_id)",
+            "CREATE INDEX IF NOT EXISTS idx_memuron_fs_entries_child ON memuron_fs_entries(child_id)",
+            "CREATE INDEX IF NOT EXISTS idx_memuron_fs_entries_scope ON memuron_fs_entries(org_token, space_token)",
+            "CREATE INDEX IF NOT EXISTS idx_memuron_fs_edges_source ON memuron_fs_edges(org_token, space_token, source_id)",
+            "CREATE INDEX IF NOT EXISTS idx_memuron_fs_edges_target ON memuron_fs_edges(org_token, space_token, target_id)",
+        ):
+            sql_store_execute(store, statement)
+        self._add_column_if_missing(
+            store,
+            "memuron_fs_entries",
+            "inherit_parent_scope BOOLEAN NOT NULL DEFAULT TRUE",
+        )
+        self._ensure_fulltext(store)
+        self._backfill_document_containers(store)
+
+    def _add_column_if_missing(self, store: Any, table: str, column_sql: str) -> None:
+        try:
+            sql_store_execute(store, f"ALTER TABLE {table} ADD COLUMN {column_sql}")
+        except Exception:
+            pass
+
+    def _ensure_fulltext(self, store: Any) -> None:
+        try:
+            sql_store_execute(
+                store,
+                "ALTER TABLE memuron_fs_nodes ADD COLUMN IF NOT EXISTS content_tsv tsvector",
+            )
+            sql_store_execute(
+                store,
+                """
+                UPDATE memuron_fs_nodes
+                SET content_tsv = to_tsvector('english', coalesce(content, ''))
+                WHERE content_tsv IS NULL
+                """,
+            )
+            sql_store_execute(
+                store,
+                """
+                CREATE INDEX IF NOT EXISTS idx_memuron_fs_nodes_content_tsv
+                ON memuron_fs_nodes USING GIN (content_tsv)
+                """,
+            )
+            setattr(store, "_memuron_fs_fulltext_ready", True)
+        except Exception:
+            setattr(store, "_memuron_fs_fulltext_ready", False)
+
+    def _backfill_document_containers(self, store: Any) -> None:
+        if not sql_store_has_tables(store):
+            return
+        rows = sql_store_fetchall(
+            store,
+            """
+            SELECT p.parent_id, p.child_id, p.metadata_json
+            FROM memuron_fs_entries p
+            JOIN memuron_fs_nodes n ON n.node_id = p.parent_id
+            WHERE n.encoding = 'document_collection'
+            """,
+        )
+        for row in rows:
+            metadata = _json(row.get("metadata_json"), {})
+            self._annotate_document_container(
+                store,
+                parent_id=str(row["parent_id"]),
+                child_id=str(row["child_id"]),
+                metadata=metadata if isinstance(metadata, dict) else {},
+            )
+
+    def apply(self, event: dict[str, object], store: Any) -> None:
+        event_type = str(event.get("event_type", ""))
+        if event_type in {"memory.created", "memory.updated", "collection.created"}:
+            self._upsert_node(event, store)
+        elif event_type in {"delete", "memory.deleted"} and event.get("subject_type") == "memory":
+            self._delete_node(str(event["subject_id"]), store)
+        elif event_type == "placement.created":
+            self._upsert_placement(event, store)
+        elif event_type == "placement.removed" or (
+            event_type == "delete" and event.get("subject_type") == "memory_placement"
+        ):
+            self._delete_placement(self._removed_id(event, "placement_id"), store)
+        elif event_type == "link.created":
+            self._upsert_link(event, store)
+        elif event_type == "link.removed" or (
+            event_type == "delete" and event.get("subject_type") == "memory_link"
+        ):
+            self._delete_edge(self._removed_id(event, "link_id"), store)
+
+    def _value(self, event: dict[str, object]) -> dict[str, Any] | None:
+        payload = event.get("payload")
+        if not isinstance(payload, dict):
+            return None
+        arthaanu = payload.get("arthaanu")
+        if not isinstance(arthaanu, dict):
+            return None
+        value = arthaanu.get("value")
+        return value if isinstance(value, dict) else None
+
+    def _upsert_node(self, event: dict[str, object], store: Any) -> None:
+        value = self._value(event)
+        if value is None or not isinstance(value.get("content"), str):
+            return
+        node_id = str(event["subject_id"])
+        content = str(value["content"])
+        node_type = str(value.get("node_type") or "text")
+        payload = _json(value.get("payload"), {})
+        metadata = _json(value.get("metadata"), {})
+        scope = _json(value.get("scope"), [])
+        payload_dict = payload if isinstance(payload, dict) else {}
+        metadata_dict = metadata if isinstance(metadata, dict) else {}
+        scope_list = scope if isinstance(scope, list) else []
+        org_token, space_token = _scope_parts(scope_list)
+        row = {
+            "node_id": node_id,
+            "org_token": org_token,
+            "space_token": space_token,
+            "node_type": node_type,
+            "display": _display(node_type, content, payload_dict),
+            "content": content,
+            "encoding": str(value.get("encoding") or "memory"),
+            "payload": payload_dict,
+            "metadata": metadata_dict,
+            "scope": scope_list,
+            "sequence": int(event["sequence"]),
+        }
+        if sql_store_has_tables(store):
+            sql_store_execute(
+                store,
+                """
+                INSERT INTO memuron_fs_nodes (
+                  node_id, org_token, space_token, node_type, display, content,
+                  encoding, payload_json, metadata_json, scope_json, sequence
+                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                ON CONFLICT(node_id) DO UPDATE SET
+                  org_token = excluded.org_token,
+                  space_token = excluded.space_token,
+                  node_type = excluded.node_type,
+                  display = excluded.display,
+                  content = excluded.content,
+                  encoding = excluded.encoding,
+                  payload_json = excluded.payload_json,
+                  metadata_json = excluded.metadata_json,
+                  scope_json = excluded.scope_json,
+                  sequence = excluded.sequence
+                """,
+                (
+                    node_id,
+                    org_token,
+                    space_token,
+                    node_type,
+                    row["display"],
+                    content,
+                    row["encoding"],
+                    _dump(payload_dict),
+                    _dump(metadata_dict),
+                    _dump(scope_list),
+                    row["sequence"],
+                ),
+            )
+            if getattr(store, "_memuron_fs_fulltext_ready", False):
+                sql_store_execute(
+                    store,
+                    """
+                    UPDATE memuron_fs_nodes
+                    SET content_tsv = to_tsvector('english', coalesce(?, ''))
+                    WHERE node_id = ?
+                    """,
+                    (content, node_id),
+                )
+        else:
+            store.memuron_fs_nodes[node_id] = row
+
+    def _upsert_placement(self, event: dict[str, object], store: Any) -> None:
+        value = self._value(event)
+        if value is None:
+            return
+        parent_id = value.get("parent_id")
+        child_id = value.get("child_id")
+        name = value.get("name")
+        if not all(isinstance(item, str) for item in (parent_id, child_id, name)):
+            return
+        placement_id = str(event["subject_id"])
+        metadata = _json(value.get("metadata"), {})
+        inherit_parent_scope = bool(value.get("inherit_parent_scope", True))
+        org_token, space_token = _scope_parts(value.get("scope"))
+        if not space_token:
+            org_token, space_token = self._node_scope(store, str(child_id))
+        row = {
+            "placement_id": placement_id,
+            "parent_id": str(parent_id),
+            "child_id": str(child_id),
+            "name": str(name),
+            "org_token": org_token,
+            "space_token": space_token,
+            "inherit_parent_scope": inherit_parent_scope,
+            "metadata": metadata if isinstance(metadata, dict) else {},
+            "sequence": int(event["sequence"]),
+        }
+        if sql_store_has_tables(store):
+            sql_store_execute(
+                store,
+                """
+                INSERT INTO memuron_fs_entries (
+                  placement_id, parent_id, child_id, name, org_token, space_token,
+                  inherit_parent_scope, metadata_json, sequence
+                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+                ON CONFLICT(placement_id) DO UPDATE SET
+                  parent_id = excluded.parent_id,
+                  child_id = excluded.child_id,
+                  name = excluded.name,
+                  org_token = excluded.org_token,
+                  space_token = excluded.space_token,
+                  inherit_parent_scope = excluded.inherit_parent_scope,
+                  metadata_json = excluded.metadata_json,
+                  sequence = excluded.sequence
+                """,
+                (
+                    placement_id,
+                    parent_id,
+                    child_id,
+                    name,
+                    org_token,
+                    space_token,
+                    inherit_parent_scope,
+                    _dump(row["metadata"]),
+                    row["sequence"],
+                ),
+            )
+            self._upsert_placement_edge(row, store)
+            self._annotate_document_container(
+                store,
+                parent_id=str(parent_id),
+                child_id=str(child_id),
+                metadata=row["metadata"],
+            )
+        else:
+            store.memuron_fs_entries[placement_id] = row
+            store.memuron_fs_edges[placement_id] = {
+                "edge_id": placement_id,
+                "source_id": str(parent_id),
+                "target_id": str(child_id),
+                "edge_type": "placement",
+                "description": str(name),
+                "org_token": org_token,
+                "space_token": space_token,
+                "inherit_parent_scope": inherit_parent_scope,
+                "sequence": row["sequence"],
+            }
+            self._annotate_document_container(
+                store,
+                parent_id=str(parent_id),
+                child_id=str(child_id),
+                metadata=row["metadata"],
+            )
+
+    def _annotate_document_container(
+        self,
+        store: Any,
+        *,
+        parent_id: str,
+        child_id: str,
+        metadata: dict[str, Any],
+    ) -> None:
+        if metadata.get("role") != "source":
+            return
+        if sql_store_has_tables(store):
+            rows = sql_store_fetchall(
+                store,
+                """
+                SELECT encoding, payload_json
+                FROM memuron_fs_nodes
+                WHERE node_id = ?
+                """,
+                (parent_id,),
+            )
+            if not rows or rows[0].get("encoding") != "document_collection":
+                return
+            payload = _json(rows[0].get("payload_json"), {})
+            payload_dict = payload if isinstance(payload, dict) else {}
+            descriptor = {
+                "id": child_id,
+                "file_name": payload_dict.get("name"),
+                "source_type": payload_dict.get("source_type"),
+                "chunk_count": payload_dict.get("chunk_count", 0),
+                "image_count": payload_dict.get("image_count", 0),
+                "graph_image_count": payload_dict.get("graph_image_count", 0),
+            }
+            updated = {
+                **payload_dict,
+                "collection_kind": "document",
+                "document_id": child_id,
+                "document": descriptor,
+            }
+            sql_store_execute(
+                store,
+                "UPDATE memuron_fs_nodes SET payload_json = ? WHERE node_id = ?",
+                (_dump(updated), parent_id),
+            )
+            return
+        row = getattr(store, "memuron_fs_nodes", {}).get(parent_id)
+        if not row or row.get("encoding") != "document_collection":
+            return
+        payload = dict(row.get("payload") or {})
+        row["payload"] = {
+            **payload,
+            "collection_kind": "document",
+            "document_id": child_id,
+            "document": {
+                "id": child_id,
+                "file_name": payload.get("name"),
+                "source_type": payload.get("source_type"),
+                "chunk_count": payload.get("chunk_count", 0),
+                "image_count": payload.get("image_count", 0),
+                "graph_image_count": payload.get("graph_image_count", 0),
+            },
+        }
+
+    def _upsert_placement_edge(self, row: dict[str, Any], store: Any) -> None:
+        sql_store_execute(
+            store,
+            """
+            INSERT INTO memuron_fs_edges (
+              edge_id, source_id, target_id, edge_type, description,
+              org_token, space_token, sequence
+            ) VALUES (?, ?, ?, 'placement', ?, ?, ?, ?)
+            ON CONFLICT(edge_id) DO UPDATE SET
+              source_id = excluded.source_id,
+              target_id = excluded.target_id,
+              edge_type = excluded.edge_type,
+              description = excluded.description,
+              org_token = excluded.org_token,
+              space_token = excluded.space_token,
+              sequence = excluded.sequence
+            """,
+            (
+                row["placement_id"],
+                row["parent_id"],
+                row["child_id"],
+                row["name"],
+                row["org_token"],
+                row["space_token"],
+                row["sequence"],
+            ),
+        )
+
+    def _upsert_link(self, event: dict[str, object], store: Any) -> None:
+        value = self._value(event)
+        if value is None:
+            return
+        source_id = value.get("source_id")
+        target_id = value.get("target_id")
+        description = value.get("description")
+        if not isinstance(source_id, str) or not isinstance(target_id, str):
+            return
+        edge_id = str(event["subject_id"])
+        org_token, space_token = self._node_scope(store, source_id)
+        row = {
+            "edge_id": edge_id,
+            "source_id": source_id,
+            "target_id": target_id,
+            "edge_type": "semantic_link",
+            "description": str(description or ""),
+            "org_token": org_token,
+            "space_token": space_token,
+            "sequence": int(event["sequence"]),
+        }
+        if sql_store_has_tables(store):
+            sql_store_execute(
+                store,
+                """
+                INSERT INTO memuron_fs_edges (
+                  edge_id, source_id, target_id, edge_type, description,
+                  org_token, space_token, sequence
+                ) VALUES (?, ?, ?, 'semantic_link', ?, ?, ?, ?)
+                ON CONFLICT(edge_id) DO UPDATE SET
+                  source_id = excluded.source_id,
+                  target_id = excluded.target_id,
+                  edge_type = excluded.edge_type,
+                  description = excluded.description,
+                  org_token = excluded.org_token,
+                  space_token = excluded.space_token,
+                  sequence = excluded.sequence
+                """,
+                (
+                    edge_id,
+                    source_id,
+                    target_id,
+                    row["description"],
+                    org_token,
+                    space_token,
+                    row["sequence"],
+                ),
+            )
+        else:
+            store.memuron_fs_edges[edge_id] = row
+
+    def _node_scope(self, store: Any, node_id: str) -> tuple[str, str]:
+        if sql_store_has_tables(store):
+            rows = sql_store_fetchall(
+                store,
+                "SELECT org_token, space_token FROM memuron_fs_nodes WHERE node_id = ?",
+                (node_id,),
+            )
+            if rows:
+                return str(rows[0].get("org_token") or ""), str(rows[0].get("space_token") or "")
+            return "", ""
+        row = getattr(store, "memuron_fs_nodes", {}).get(node_id, {})
+        return str(row.get("org_token") or ""), str(row.get("space_token") or "")
+
+    def _removed_id(self, event: dict[str, object], key: str) -> str:
+        payload = event.get("payload")
+        if isinstance(payload, dict) and payload.get(key):
+            return str(payload[key])
+        return str(event.get("subject_id") or "")
+
+    def reset(self, store: Any) -> None:
+        if sql_store_has_tables(store):
+            sql_store_execute(store, "DELETE FROM memuron_fs_edges")
+            sql_store_execute(store, "DELETE FROM memuron_fs_entries")
+            sql_store_execute(store, "DELETE FROM memuron_fs_nodes")
+        else:
+            store.memuron_fs_nodes = {}
+            store.memuron_fs_entries = {}
+            store.memuron_fs_edges = {}
+
+    def _delete_node(self, node_id: str, store: Any) -> None:
+        if sql_store_has_tables(store):
+            sql_store_execute(
+                store,
+                "DELETE FROM memuron_fs_edges WHERE source_id = ? OR target_id = ?",
+                (node_id, node_id),
+            )
+            sql_store_execute(
+                store,
+                "DELETE FROM memuron_fs_entries WHERE parent_id = ? OR child_id = ?",
+                (node_id, node_id),
+            )
+            sql_store_execute(store, "DELETE FROM memuron_fs_nodes WHERE node_id = ?", (node_id,))
+        else:
+            store.memuron_fs_nodes.pop(node_id, None)
+            store.memuron_fs_entries = {
+                key: row
+                for key, row in store.memuron_fs_entries.items()
+                if node_id not in {row["parent_id"], row["child_id"]}
+            }
+            store.memuron_fs_edges = {
+                key: row
+                for key, row in store.memuron_fs_edges.items()
+                if node_id not in {row["source_id"], row["target_id"]}
+            }
+
+    def _delete_placement(self, placement_id: str, store: Any) -> None:
+        if not placement_id:
+            return
+        if sql_store_has_tables(store):
+            sql_store_execute(
+                store, "DELETE FROM memuron_fs_entries WHERE placement_id = ?", (placement_id,)
+            )
+            sql_store_execute(store, "DELETE FROM memuron_fs_edges WHERE edge_id = ?", (placement_id,))
+        else:
+            store.memuron_fs_entries.pop(placement_id, None)
+            store.memuron_fs_edges.pop(placement_id, None)
+
+    def _delete_edge(self, edge_id: str, store: Any) -> None:
+        if not edge_id:
+            return
+        if sql_store_has_tables(store):
+            sql_store_execute(store, "DELETE FROM memuron_fs_edges WHERE edge_id = ?", (edge_id,))
+        else:
+            store.memuron_fs_edges.pop(edge_id, None)