okfgraph 0.2.4__py3-none-any.whl

okfgraph/components/lint.py

@@ -0,0 +1,118 @@
+"""Pre-import bundle gate: frontmatter + link validation, no DB, no model.
+
+``lint_bundle()`` checks the file side (what ``import --all`` is *about*
+to ingest); doctor checks the row side (what is *already* indexed).
+Same ``links.py`` pure helpers, same resolution rules as import —
+a lint-clean bundle must import with zero ``broken_link`` findings
+(locked by the consistency test in ``tests/test_lint.py``).
+
+Report shape (JSON-stable, all lists sorted)::
+    {"dir": str, "files": int,
+     "errors": [{"file", "rule", "message", ...}],
+     "warnings": [{"file", "rule", "message"}],
+     "clean": bool}  # True when errors is empty (warnings allowed)
+"""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import Any, Dict, List, Tuple
+
+import frontmatter
+
+from okfgraph.components.import_ import is_concept_file, parse_source_file
+from okfgraph.components.links import (
+    build_name_index,
+    extract_md_links,
+    extract_wikilinks,
+    is_external,
+    normalize_path_link,
+    resolve_wiki,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def _err(file: str, rule: str, message: str, **extra: Any) -> Dict[str, Any]:
+    item: Dict[str, Any] = {"file": file, "rule": rule, "message": message}
+    item.update(extra)
+    return item
+
+
+def lint_bundle(bundle_dir: str | Path) -> Dict[str, Any]:
+    """Validate a bundle directory without touching any database or model."""
+    root = Path(bundle_dir)
+    files = sorted(fp for fp in root.rglob("*") if is_concept_file(fp))
+
+    errors: List[Dict[str, Any]] = []
+    warnings: List[Dict[str, Any]] = []
+    parsed: List[Tuple[str, Any, str, str]] = []  # (cid, concept, body, rel)
+
+    for fp in files:
+        rel = str(fp.relative_to(root)).replace("\\", "/")
+        try:
+            concept, body, cid = parse_source_file(fp, root)
+        except Exception as e:  # noqa: BLE001 — every failure mode is a finding
+            errors.append(_err(rel, "parse", f"{type(e).__name__}: {e}"))
+            continue
+        try:
+            raw_fm = dict(frontmatter.load(fp).metadata)
+        except Exception as e:  # noqa: BLE001 — same finding class as above
+            errors.append(_err(rel, "parse", f"{type(e).__name__}: {e}"))
+            continue
+        # Import synthesizes both (note / stem), so these warn — erroring
+        # would contradict import behaviour (deliberate deviation from
+        # google-okf, whose pipeline has no synthesis step).
+        if not raw_fm.get("type"):
+            warnings.append(_err(
+                rel, "missing_type", "no 'type:' frontmatter; will import as 'note'"))
+        if not raw_fm.get("title"):
+            warnings.append(_err(
+                rel, "missing_title", "no 'title:' frontmatter; will use filename stem"))
+        parsed.append((cid, concept, body, rel))
+
+    # Resolution context mirrors ImportManager: known ids + name index.
+    known_ids = {cid for cid, _, _, _ in parsed}
+    index_entries = []
+    for cid, concept, _, _ in parsed:
+        extra = concept.model_extra or {}
+        index_entries.append({
+            "id": cid,
+            "title": concept.title,
+            "uid": extra.get("uid"),
+            "aliases": extra.get("aliases", extra.get("alias")),
+        })
+    maps, _ambiguous = build_name_index(index_entries)
+
+    for cid, _concept, body, rel in parsed:
+        # Same extraction + skip rules as _resolve_body_links: only
+        # *.md-anchored targets match; externals never resolve-or-fail.
+        for raw in extract_md_links(body):
+            if is_external(raw):
+                continue
+            target = normalize_path_link(raw.split("#", 1)[0])
+            if target not in known_ids:
+                errors.append(_err(
+                    rel, "dangling_link",
+                    f"would import as BrokenLink (target has no concept)",
+                    link=raw, target=target or "(empty)"))
+        for raw in extract_wikilinks(body):
+            if not raw or is_external(raw):
+                continue
+            if resolve_wiki(raw, maps, known_ids) is None:
+                errors.append(_err(
+                    rel, "dangling_wikilink",
+                    "would import as BrokenLink (name resolves to nothing; "
+                    "ambiguous names never resolve)",
+                    link=raw))
+
+    errors.sort(key=lambda e: (e["file"], e["rule"], e["message"]))
+    warnings.sort(key=lambda w: (w["file"], w["rule"], w["message"]))
+    return {
+        "dir": str(root),
+        "files": len(files),
+        "errors": errors,
+        "warnings": warnings,
+        "clean": not errors,
+    }

okfgraph/components/purge.py

@@ -0,0 +1,342 @@
+"""PurgeManager — hard purge, soft-delete, and recovery-window management.
+
+Extracted from OKFRouter (purge + soft-delete-with-recovery sections).
+"""
+
+from __future__ import annotations
+import logging
+from datetime import datetime, timezone, timedelta
+from typing import Any, Callable, Dict, List, Optional, Tuple
+
+from okfgraph.models import ChunkModel, ConceptModel
+
+logger = logging.getLogger(__name__)
+
+class PurgeManager:
+    def __init__(self, conn, write_lock_ctx: Callable):
+        self.conn = conn
+        self._write_lock_ctx = write_lock_ctx
+
+    SOFT_DELETE_WINDOW = 24 * 60 * 60
+
+    def _purge_concept(self, concept_id: str) -> bool:
+        """Delete a concept and all its dependents from the graph.
+
+        Removes:
+        - The Concept node (and its embedding)
+        - All Chunk nodes linked via PART_OF
+        - All LINKS_TO relationships (incoming and outgoing)
+        - All INCLUDES_ASSET relationships
+        - ImageAsset nodes that have **zero remaining** INCLUDES_ASSET
+          edges (orphan check — shared assets are preserved)
+        - BrokenLink entries where this concept was source or target
+        - FileHash entry for the concept's file path
+
+        Args:
+            concept_id: The ID of the concept to purge.
+
+        Returns:
+            True if a concept was found and purged, False if not found.
+        """
+        # Check if the concept exists
+        rows = self.conn.execute(
+            "MATCH (c:Concept {id: $id}) RETURN count(c) AS cnt",
+            {"id": concept_id},
+        ).rows_as_dict().get_all()
+        if not rows or rows[0]["cnt"] == 0:
+            logger.debug("purge: concept %s not found, skipping", concept_id)
+            return False
+
+        self.conn.execute("BEGIN TRANSACTION")
+        try:
+            # 1. Find and sever INCLUDES_ASSET edges, collecting asset IDs.
+            asset_rows = self.conn.execute(
+                """
+                MATCH (c:Concept {id: $id})-[:INCLUDES_ASSET]->(i:ImageAsset)
+                RETURN i.id AS aid
+                """,
+                {"id": concept_id},
+            ).rows_as_dict().get_all()
+            asset_ids = [r["aid"] for r in asset_rows]
+
+            # 2. Delete all Chunk nodes for this concept (DETACH DELETE on
+            #    Concept doesn't cascade to Chunk — they are separate nodes).
+            self.conn.execute(
+                """
+                MATCH (ch:Chunk {parent_doc_id: $id})
+                DETACH DELETE ch
+                """,
+                {"id": concept_id},
+            )
+
+            # 3. DETACH DELETE the Concept (cascades LINKS_TO, CONTAINS,
+            #    INCLUDES_ASSET edges).
+            self.conn.execute(
+                "MATCH (c:Concept {id: $id}) DETACH DELETE c",
+                {"id": concept_id},
+            )
+
+            # 4. Clean up orphaned ImageAssets — only delete if no other
+            #    Concept still references them. This handles the case where
+            #    two files share the same okf-asset://<id> URI.
+            for aid in asset_ids:
+                ref_count = self.conn.execute(
+                    """
+                    MATCH (i:ImageAsset {id: $aid})
+                    OPTIONAL MATCH (i)<-[:INCLUDES_ASSET]-(other:Concept)
+                    RETURN count(other) AS refs
+                    """,
+                    {"aid": aid},
+                ).rows_as_dict().get_all()
+                if ref_count and ref_count[0]["refs"] == 0:
+                    self.conn.execute(
+                        "MATCH (i:ImageAsset {id: $aid}) DELETE i",
+                        {"aid": aid},
+                    )
+                    logger.debug("purge: deleted orphaned asset %s", aid)
+
+            # 5. Clean up BrokenLink entries where this concept was source
+            #    or target.
+            self.conn.execute(
+                """
+                MATCH (bl:BrokenLink)
+                WHERE bl.source_id = $id OR bl.target_id = $id
+                DELETE bl
+                """,
+                {"id": concept_id},
+            )
+
+            # 6. Remove FileHash entry for this concept.
+            self.conn.execute(
+                """
+                MATCH (f:FileHash {concept_id: $id})
+                DELETE f
+                """,
+                {"id": concept_id},
+            )
+
+            self.conn.execute("COMMIT")
+            logger.info("purge: deleted concept %s", concept_id)
+            return True
+
+        except Exception:
+            try:
+                self.conn.execute("ROLLBACK")
+            except Exception:
+                pass
+            raise
+
+
+    def _soft_delete_concept(self, concept_id: str) -> bool:
+        """Soft-delete a concept by moving it to the DeletedConcept table.
+
+        The concept is preserved with a timestamp so it can be recovered
+        within the recovery window. After the window expires, the concept
+        is permanently deleted.
+
+        Args:
+            concept_id: The ID of the concept to soft-delete.
+
+        Returns:
+            True if a concept was found and soft-deleted, False if not found.
+        """
+        # Acquire write lock (Gap #7b)
+        with self._write_lock_ctx():
+            return self._soft_delete_concept_inner(concept_id)
+
+
+    def _soft_delete_concept_inner(self, concept_id: str) -> bool:
+        """Inner implementation (called under write lock)."""
+        # Check if the concept exists
+        rows = self.conn.execute(
+            "MATCH (c:Concept {id: $id}) RETURN count(c) AS cnt",
+            {"id": concept_id},
+        ).rows_as_dict().get_all()
+        if not rows or rows[0]["cnt"] == 0:
+            logger.debug("soft_delete: concept %s not found, skipping", concept_id)
+            return False
+
+        # Fetch concept data for preservation
+        concept_rows = self.conn.execute(
+            """
+            MATCH (c:Concept {id: $id})
+            RETURN c.title AS title, c.body AS body, c.type AS ctype, c.tags AS tags
+            """,
+            {"id": concept_id},
+        ).rows_as_dict().get_all()
+        if not concept_rows:
+            return False
+
+        concept_data = concept_rows[0]
+        deleted_at = datetime.now().isoformat()
+
+        # Store in DeletedConcept table
+        self.conn.execute(
+            """
+            INSERT INTO DeletedConcept (id, original_id, title, body, deleted_at, type, tags)
+            VALUES ($id, $oid, $title, $body, $deleted_at, $type, $tags)
+            """,
+            {
+                "id": f"deleted_{concept_id}_{int(time.time())}",
+                "oid": concept_id,
+                "title": concept_data.get("title", ""),
+                "body": concept_data.get("body", ""),
+                "deleted_at": deleted_at,
+                "type": concept_data.get("ctype", ""),
+                "tags": json.dumps(concept_data.get("tags", [])),
+            },
+        )
+
+        # Now perform the hard delete
+        self._purge_concept(concept_id)
+
+        logger.info(
+            "soft_delete: concept %s moved to DeletedConcept (recovery until %s)",
+            concept_id,
+            (datetime.now() + __import__("datetime").timedelta(seconds=self.SOFT_DELETE_WINDOW)).isoformat(),
+        )
+        return True
+
+
+    def _recover_concept(self, concept_id: str) -> bool:
+        """Recover a soft-deleted concept from the DeletedConcept table.
+
+        Args:
+            concept_id: The original ID of the concept to recover.
+
+        Returns:
+            True if the concept was recovered, False if not found.
+        """
+        # Acquire write lock (Gap #7b)
+        with self._write_lock_ctx():
+            return self._recover_concept_inner(concept_id)
+
+
+    def _recover_concept_inner(self, concept_id: str) -> bool:
+        """Inner implementation (called under write lock)."""
+        # Check if the concept exists in DeletedConcept
+        rows = self.conn.execute(
+            """
+            MATCH (d:DeletedConcept {original_id: $id})
+            RETURN d.title AS title, d.body AS body, d.type AS ctype, d.tags AS tags, d.deleted_at AS deleted_at
+            """,
+            {"id": concept_id},
+        ).rows_as_dict().get_all()
+
+        if not rows:
+            logger.debug("recover: concept %s not found in DeletedConcept", concept_id)
+            return False
+
+        deleted_at = datetime.fromisoformat(rows[0]["deleted_at"])
+        now = datetime.now()
+        if (now - deleted_at).total_seconds() > self.SOFT_DELETE_WINDOW:
+            logger.warning(
+                "recover: concept %s deleted at %s is past the recovery window (%ds)",
+                concept_id,
+                deleted_at.isoformat(),
+                self.SOFT_DELETE_WINDOW,
+            )
+            return False
+
+        # Restore the concept
+        concept_data = rows[0]
+        tags = json.loads(concept_data["tags"]) if isinstance(concept_data["tags"], str) else concept_data["tags"]
+
+        self.conn.execute(
+            """
+            INSERT INTO Concept (id, title, body, type, tags)
+            VALUES ($id, $title, $body, $type, $tags)
+            """,
+            {
+                "id": concept_id,
+                "title": concept_data["title"],
+                "body": concept_data["body"],
+                "type": concept_data["ctype"],
+                "tags": tags,
+            },
+        )
+
+        # Remove from DeletedConcept
+        self.conn.execute(
+            "MATCH (d:DeletedConcept {original_id: $id}) DELETE d",
+            {"id": concept_id},
+        )
+
+        logger.info("recover: concept %s restored from DeletedConcept", concept_id)
+        return True
+
+
+    def list_deleted_concepts(self) -> List[Dict[str, Any]]:
+        """List all soft-deleted concepts with recovery status.
+
+        Returns:
+            List of dicts with concept_id, title, deleted_at, and recoverable status.
+        """
+        rows = self.conn.execute(
+            """
+            MATCH (d:DeletedConcept)
+            RETURN d.original_id AS id, d.title AS title, d.deleted_at AS deleted_at, d.type AS type
+            ORDER BY d.deleted_at DESC
+            """
+        ).rows_as_dict().get_all()
+
+        now = datetime.now()
+        results = []
+        for row in rows:
+            deleted_at = datetime.fromisoformat(row["deleted_at"])
+            age_seconds = (now - deleted_at).total_seconds()
+            results.append({
+                "concept_id": row["id"],
+                "title": row["title"],
+                "type": row["type"],
+                "deleted_at": row["deleted_at"],
+                "age_seconds": age_seconds,
+                "recoverable": age_seconds <= self.SOFT_DELETE_WINDOW,
+            })
+        return results
+
+
+    def purge_deleted_concepts(self, older_than: Optional[int] = None) -> int:
+        """Permanently delete soft-deleted concepts past the recovery window.
+
+        Args:
+            older_than: Optional override for the recovery window (seconds).
+                        Defaults to SOFT_DELETE_WINDOW.
+
+        Returns:
+            Number of concepts permanently deleted.
+        """
+        # Acquire write lock (Gap #7b)
+        with self._write_lock_ctx():
+            return self._purge_deleted_concepts_inner(older_than)
+
+
+    def _purge_deleted_concepts_inner(self, older_than: Optional[int]) -> int:
+        """Inner implementation (called under write lock)."""
+        threshold = older_than if older_than is not None else self.SOFT_DELETE_WINDOW
+        cutoff = datetime.now() - __import__("datetime").timedelta(seconds=threshold)
+
+        # Find expired entries
+        rows = self.conn.execute(
+            """
+            MATCH (d:DeletedConcept)
+            WHERE d.deleted_at < $cutoff
+            RETURN d.original_id AS id
+            """,
+            {"cutoff": cutoff.isoformat()},
+        ).rows_as_dict().get_all()
+
+        count = 0
+        for row in rows:
+            # Permanently delete (hard purge)
+            if self._purge_concept(row["id"]):
+                count += 1
+            # Remove from DeletedConcept
+            self.conn.execute(
+                "MATCH (d:DeletedConcept {original_id: $id}) DELETE d",
+                {"id": row["id"]},
+            )
+
+        logger.info("purge_deleted: permanently deleted %d expired concept(s)", count)
+        return count
+

okfgraph/components/ranking.py

@@ -0,0 +1,168 @@
+"""Model-free retrieval: lexical seeds + exact Personalized PageRank.
+
+Mirrors the okf-ingest ``seeds`` → multi-seed PPR cascade: a query is scored
+lexically against concept metadata, the top seeds weight an exact
+power-iteration PPR over the resolved ``LINKS_TO`` graph, and the ranked
+concepts come back — with no embedding model loaded. Over a sparse graph this
+degrades gracefully to seed order (== the lexical baseline).
+
+Determinism rules (all load-bearing for the golden fixtures):
+  - seed ties break by concept id ascending;
+  - edges iterate in sorted ``(src, dst)`` index order;
+  - the L1 convergence sum accumulates in ascending node order;
+  - scores round half-even to 10 decimals (Python ``round``).
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any, Dict, List, Mapping, Optional, Sequence, Set, Tuple
+
+TOKEN_RE = re.compile(r"[a-z0-9]+")
+
+#: Fixed stopword list — keep in sync with tests/fixtures, never grow silently.
+STOPWORDS = frozenset({
+    "the", "and", "for", "are", "was", "were", "with", "that", "this",
+    "from", "how", "what", "when", "where", "which", "does", "did",
+    "can", "could", "should", "would", "will", "has", "have", "had",
+    "not", "its", "our", "your", "their", "about", "into", "over",
+    "under", "why", "who", "whom",
+})
+
+TITLE_HIT = 3
+META_HIT = 2   # description or tags
+BODY_HIT = 1
+
+
+def query_tokens(query: str) -> List[str]:
+    """Lowercase alphanumeric tokens, len >= 3, minus stopwords (sorted)."""
+    return sorted({
+        m for m in TOKEN_RE.findall((query or "").lower())
+        if len(m) >= 3 and m not in STOPWORDS
+    })
+
+
+def seeds(
+    concepts: Sequence[Mapping[str, Any]],
+    query: str,
+    k: int = 20,
+) -> List[Tuple[str, float]]:
+    """Lexically score concepts against the query.
+
+    ``+3`` per distinct token in the title, ``+2`` in description/tags,
+    ``+1`` in the body. Returns ``(concept_id, score)`` sorted by score
+    desc, id asc, truncated at ``k``. Pure function — no DB, no model.
+    """
+    toks = query_tokens(query)
+    if not toks:
+        return []
+    out: List[Tuple[str, float]] = []
+    for c in concepts:
+        cid = c.get("id", "")
+        if not cid:
+            continue
+        title = str(c.get("title") or "").lower()
+        desc = str(c.get("description") or "").lower()
+        tags = c.get("tags") or []
+        tags_text = " ".join(str(t) for t in tags).lower() if isinstance(tags, list) else str(tags).lower()
+        body = str(c.get("body") or "").lower()
+        score = 0
+        for t in toks:
+            if t in title:
+                score += TITLE_HIT
+            if t in desc or t in tags_text:
+                score += META_HIT
+            if t in body:
+                score += BODY_HIT
+        if score > 0:
+            out.append((cid, float(score)))
+    out.sort(key=lambda kv: (-kv[1], kv[0]))
+    return out[:k]
+
+
+def ppr(
+    nodes: Sequence[str],
+    directed_edges: Sequence[Tuple[str, str]],
+    starts: Sequence[str],
+    weights: Optional[Sequence[float]] = None,
+    damping: float = 0.85,
+    tol: float = 1e-12,
+    max_iter: int = 200,
+    k: Optional[int] = None,
+) -> List[Tuple[str, float]]:
+    """Exact Personalized PageRank over the undirected link graph.
+
+    ``nodes`` need not be sorted (sorted internally); ``directed_edges`` are
+    symmetrized, self-loops dropped. Dangling mass returns to the seed
+    distribution. Unknown start ids raise ``KeyError``.
+    """
+    ids = sorted(set(nodes))
+    idx = {nid: i for i, nid in enumerate(ids)}
+    missing = [s for s in starts if s not in idx]
+    if missing:
+        raise KeyError(f"start concept not found: {', '.join(missing)}")
+    w = list(weights) if weights is not None else [1.0] * len(starts)
+    if len(w) != len(starts) or sum(w) <= 0 or any(x < 0 for x in w):
+        raise ValueError("weights must be non-negative, same length as starts, positive sum")
+
+    n = len(ids)
+    edges: Set[Tuple[int, int]] = set()
+    for s, d in directed_edges:
+        if s == d or s not in idx or d not in idx:
+            continue
+        edges.add((idx[s], idx[d]))
+        edges.add((idx[d], idx[s]))
+    ordered = sorted(edges)
+    deg = [0] * n
+    for s, _ in ordered:
+        deg[s] += 1
+
+    seed = [0.0] * n
+    for s, x in zip(starts, w):
+        seed[idx[s]] += x
+    total = sum(seed)
+    seed = [x / total for x in seed]
+
+    p = list(seed)
+    for _ in range(max_iter):
+        contrib = [0.0] * n
+        for s, d in ordered:  # fixed order -> deterministic fp
+            if p[s] != 0.0:
+                contrib[d] += p[s] / deg[s]
+        dangling = sum(p[i] for i in range(n) if deg[i] == 0)
+        nxt = [
+            (1.0 - damping) * seed[i] + damping * (contrib[i] + dangling * seed[i])
+            for i in range(n)
+        ]
+        delta = sum(abs(nxt[i] - p[i]) for i in range(n))  # ascending order
+        p = nxt
+        if delta < tol:
+            break
+
+    rows = [
+        (nid, round(p[i], 10))
+        for i, nid in enumerate(ids)
+        if round(p[i], 10) > 0.0
+    ]
+    rows.sort(key=lambda kv: (-kv[1], kv[0]))
+    return rows[:k] if k is not None else rows
+
+
+def seed_ranked_ppr(
+    concepts: Sequence[Mapping[str, Any]],
+    directed_edges: Sequence[Tuple[str, str]],
+    query: str,
+    seed_k: int = 20,
+    k: Optional[int] = None,
+) -> List[Tuple[str, float]]:
+    """One-call cascade: lexical seeds (as weights) → multi-seed PPR."""
+    found = seeds(concepts, query, k=seed_k)
+    if not found:
+        return []
+    ids = [cid for cid, _ in found]
+    weights = [s for _, s in found]
+    known = {c.get("id") for c in concepts}
+    return ppr(
+        [c.get("id", "") for c in concepts if c.get("id") in known],
+        directed_edges, ids, weights, k=k,
+    )