memplex 3.2.0__py3-none-any.whl

memnex/models/task.py

@@ -0,0 +1,62 @@
+"""Task and compaction types."""
+
+from datetime import datetime
+from enum import Enum
+from dataclasses import dataclass, field
+from typing import List, Optional, Any
+
+
+class BackgroundTask(Enum):
+    EXTRACT_DOCUMENT = "extract_document"
+    BUILD_INDEX = "build_index"
+    COMPILE_WIKI = "compile_wiki"
+    REFRESH_VECTOR = "refresh_vector"
+    COMPACTION = "compaction"
+
+
+class TaskStatus(Enum):
+    PENDING = "pending"
+    RUNNING = "running"
+    COMPLETED = "completed"
+    FAILED = "failed"
+    CANCELLED = "cancelled"
+
+
+@dataclass
+class TaskInfo:
+    task_id: str
+    task_type: BackgroundTask
+    status: TaskStatus
+    created_at: datetime
+    completed_at: Optional[datetime] = None
+    payload: Optional[dict] = None
+    result: Any = None
+    error: Optional[str] = None
+    retry_count: int = 0
+    max_retries: int = 3
+
+
+class CompactionScope(Enum):
+    SESSION = "session"
+    PROJECT = "project"
+    GLOBAL = "global"
+
+
+@dataclass
+class CompactionStageResult:
+    stage: str
+    processed: int
+    removed: int
+    merged: int
+    duration_ms: int
+    abort: bool = False
+
+
+@dataclass
+class CompactionResult:
+    total_processed: int
+    total_removed: int
+    total_merged: int
+    duration_ms: int
+    stages: List[CompactionStageResult] = field(default_factory=list)
+    skipped: bool = False

memnex/processing/__init__.py

@@ -0,0 +1 @@
+"""Data processing pipeline: association, merging, graph building."""

memnex/processing/graph_builder.py

@@ -0,0 +1,278 @@
+"""GraphBuilder -- construct graph edges from Function nodes.
+
+Detects REFERENCES, DEPENDS_ON, CONFLICTS_WITH, ASSOCIATED_WITH, and
+BELONGS_TO edges by analysing cross-references, name patterns, and
+domain membership.
+
+Works with :class:`MemoryStore` for persistence, unlike the legacy
+``merger/graph_builder.py`` which was single-run only.
+
+Usage::
+
+    builder = GraphBuilder(store, config)
+    edges = builder.process(func, existing_graph)
+    builder.build_from_batch(functions)
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from datetime import datetime
+from typing import List, Optional, Set, TYPE_CHECKING
+
+from memnex.models import (
+    EdgeType,
+    Function,
+    GraphData,
+    GraphEdge,
+)
+
+if TYPE_CHECKING:
+    from memnex.config import MemNexConfig, GraphConfig
+    from memnex.storage.base import MemoryStore
+
+logger = logging.getLogger(__name__)
+
+
+class GraphBuilder:
+    """Build graph edges for Function nodes.
+
+    Parameters
+    ----------
+    store:
+        Active :class:`MemoryStore` backend (used for name lookups).
+    config:
+        Optional :class:`MemNexConfig` (reads ``graph`` sub-config).
+    """
+
+    def __init__(
+        self,
+        store: MemoryStore,
+        config: Optional[MemNexConfig] = None,
+    ) -> None:
+        self._store = store
+        self._config = config
+        self._graph_config: Optional[GraphConfig] = (
+            config.graph if config else None
+        )
+
+    # ── Public API ──────────────────────────────────────────────────
+
+    def process(
+        self,
+        func: Function,
+        existing_graph: Optional[GraphData] = None,
+    ) -> List[GraphEdge]:
+        """Detect and return edges for a single Function.
+
+        Parameters
+        ----------
+        func:
+            The Function to analyse.
+        existing_graph:
+            Current graph state (used to avoid duplicate edges and
+            to look up neighbour nodes).  If ``None``, edges are
+            computed from scratch.
+        """
+        edges: List[GraphEdge] = []
+        existing_set = self._edge_set(existing_graph)
+
+        # 1. REFERENCES -- from cross_references field
+        for ref in func.cross_references:
+            target_id = ref.get("target_id", "") if isinstance(ref, dict) else ""
+            target_name = ref.get("target", "") if isinstance(ref, dict) else str(ref)
+            if target_id:
+                edge = self._make_edge(
+                    source=func.id,
+                    target=target_id,
+                    edge_type=EdgeType.REFERENCES.value,
+                    evidence=[f"cross_reference from {func.name}"],
+                )
+            elif target_name:
+                resolved = self._resolve_by_name(target_name)
+                if resolved:
+                    edge = self._make_edge(
+                        source=func.id,
+                        target=resolved,
+                        edge_type=EdgeType.REFERENCES.value,
+                        evidence=[f"cross_reference: {func.name} -> {target_name}"],
+                    )
+                else:
+                    continue
+            else:
+                continue
+
+            if self._edge_key(edge) not in existing_set:
+                edges.append(edge)
+                existing_set.add(self._edge_key(edge))
+
+        # 2. DEPENDS_ON -- from action field references
+        all_funcs = self._get_all_funcs()
+        for other in all_funcs:
+            if other.id == func.id:
+                continue
+            if self._has_name_reference(func, other):
+                key = (func.id, other.id, EdgeType.DEPENDS_ON.value)
+                if key not in existing_set:
+                    edges.append(self._make_edge(
+                        source=func.id,
+                        target=other.id,
+                        edge_type=EdgeType.DEPENDS_ON.value,
+                        evidence=[f"{func.name} references {other.name}"],
+                    ))
+                    existing_set.add(key)
+
+        # 3. CONFLICTS_WITH -- same domain, overlapping trigger/action
+        for other in all_funcs:
+            if other.id == func.id:
+                continue
+            if self._detect_conflict(func, other):
+                key = (func.id, other.id, EdgeType.CONFLICTS_WITH.value)
+                rev_key = (other.id, func.id, EdgeType.CONFLICTS_WITH.value)
+                if key not in existing_set and rev_key not in existing_set:
+                    edges.append(self._make_edge(
+                        source=func.id,
+                        target=other.id,
+                        edge_type=EdgeType.CONFLICTS_WITH.value,
+                        evidence=[f"conflicting definitions in domain {func.domain or 'unknown'}"],
+                    ))
+                    existing_set.add(key)
+
+        # 4. BELONGS_TO -- domain membership
+        if func.domain:
+            domain_id = f"domain_{func.domain.replace(' ', '_').lower()}"
+            key = (func.id, domain_id, EdgeType.BELONGS_TO.value)
+            if key not in existing_set:
+                edges.append(self._make_edge(
+                    source=func.id,
+                    target=domain_id,
+                    edge_type=EdgeType.BELONGS_TO.value,
+                    evidence=[f"{func.name} belongs to {func.domain}"],
+                ))
+                existing_set.add(key)
+
+        # 5. ASSOCIATED_WITH -- shared domain with other functions
+        if func.domain:
+            for other in all_funcs:
+                if other.id == func.id:
+                    continue
+                if other.domain == func.domain:
+                    key = (func.id, other.id, EdgeType.ASSOCIATED_WITH.value)
+                    rev_key = (other.id, func.id, EdgeType.ASSOCIATED_WITH.value)
+                    if key not in existing_set and rev_key not in existing_set:
+                        edges.append(self._make_edge(
+                            source=func.id,
+                            target=other.id,
+                            edge_type=EdgeType.ASSOCIATED_WITH.value,
+                            weight=0.5,
+                            evidence=[f"shared domain: {func.domain}"],
+                        ))
+                        existing_set.add(key)
+
+        return edges
+
+    def build_from_batch(
+        self,
+        funcs: List[Function],
+    ) -> List[GraphEdge]:
+        """Build edges for a batch of Functions.
+
+        The graph is built incrementally: each Function sees edges
+        from previously processed Functions in the same batch.
+        """
+        all_edges: List[GraphEdge] = []
+        accumulated_graph = GraphData(nodes=[], edges=[])
+
+        for func in funcs:
+            accumulated_graph.nodes.append(func)
+            new_edges = self.process(func, accumulated_graph)
+            all_edges.extend(new_edges)
+            accumulated_graph.edges.extend(new_edges)
+
+        return all_edges
+
+    # ── Edge detection helpers ──────────────────────────────────────
+
+    def _has_name_reference(self, source: Function, target: Function) -> bool:
+        """Check if *source* mentions *target*'s name in its action field."""
+        target_name = target.name.lower()
+        if not target_name:
+            return False
+        for fv in source.action:
+            if target_name in fv.desc.lower():
+                return True
+        for fv in source.trigger:
+            if target_name in fv.desc.lower():
+                return True
+        return False
+
+    def _detect_conflict(self, a: Function, b: Function) -> bool:
+        """Detect if two Functions in the same domain conflict.
+
+        Conflict heuristic:
+        - Same domain (non-empty)
+        - Overlapping trigger descriptions (substring match)
+        """
+        if not a.domain or a.domain != b.domain:
+            return False
+        a_triggers = {fv.desc.lower() for fv in a.trigger}
+        b_triggers = {fv.desc.lower() for fv in b.trigger}
+        return bool(a_triggers & b_triggers)
+
+    # ── Utility helpers ─────────────────────────────────────────────
+
+    @staticmethod
+    def _make_edge(
+        source: str,
+        target: str,
+        edge_type: str,
+        weight: float = 1.0,
+        evidence: Optional[List[str]] = None,
+    ) -> GraphEdge:
+        return GraphEdge(
+            source=source,
+            target=target,
+            edge_type=edge_type,
+            weight=weight,
+            evidence=evidence or [],
+            created_at=datetime.now(),
+        )
+
+    @staticmethod
+    def _edge_key(edge: GraphEdge) -> tuple:
+        return (edge.source, edge.target, edge.edge_type)
+
+    @staticmethod
+    def _edge_set(graph: Optional[GraphData]) -> Set[tuple]:
+        if graph is None:
+            return set()
+        return {
+            (e.source, e.target, e.edge_type)
+            for e in graph.edges
+        }
+
+    def _resolve_by_name(self, name: str) -> Optional[str]:
+        """Look up a Function ID by its name via the store."""
+        try:
+            funcs = self._store.list_functions(limit=100000)
+            for f in funcs:
+                if f.name == name:
+                    return f.id
+        except Exception:
+            pass
+        return None
+
+    def _get_all_funcs(self) -> List[Function]:
+        """Retrieve all stored Functions (cached per build batch)."""
+        if not hasattr(self, "_funcs_cache"):
+            try:
+                self._funcs_cache = self._store.list_functions(limit=100000)
+            except Exception:
+                self._funcs_cache = []
+        return self._funcs_cache
+
+    def invalidate_cache(self) -> None:
+        """Clear the internal function list cache."""
+        if hasattr(self, "_funcs_cache"):
+            del self._funcs_cache

memnex/processing/merger/__init__.py

@@ -0,0 +1,6 @@
+"""Merge and conflict resolution modules."""
+
+from .conflict_resolver import ConflictResolver, Conflict
+from .confidence_calculator import ConfidenceCalculator
+
+__all__ = ["ConflictResolver", "Conflict", "ConfidenceCalculator"]

memnex/processing/merger/confidence_calculator.py

@@ -0,0 +1,127 @@
+"""Confidence calculation based on extraction quality signals."""
+
+from typing import List, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from memnex.models.paragraph import Paragraph, Sentence
+    from memnex.models.memory import Function
+
+
+class ConfidenceCalculator:
+    """Calculates confidence scores based on extraction quality signals."""
+
+    SOURCE_BASE = {
+        "text": 0.95,
+        "markdown": 0.95,
+        "pdf": 0.90,
+        "docx": 0.90,
+        "image": 0.85,
+        "vision": 0.80,
+        "url": 0.90,
+    }
+
+    SOURCE_ALIASES = {
+        "clipboard": "text",
+        "file:": "file",
+    }
+
+    def calculate_paragraph_confidence(
+        self,
+        para: "Paragraph",
+        source_hint: str = "text"
+    ) -> float:
+        """
+        Calculate confidence for a paragraph -> Function conversion.
+
+        Args:
+            para: The paragraph to evaluate
+            source_hint: Hint about source type
+
+        Returns:
+            Confidence score between 0.0 and 1.0
+        """
+        base = self._get_base_confidence(source_hint)
+
+        adjustments = []
+
+        if para.sentences:
+            sent_count = len(para.sentences)
+            if 2 <= sent_count <= 10:
+                adjustments.append(0.02)
+            elif sent_count > 10:
+                adjustments.append(0.01)
+        else:
+            adjustments.append(-0.05)
+
+        if para.section:
+            adjustments.append(0.03)
+
+        text_len = len(para.raw_text) if para.raw_text else 0
+        if text_len < 10:
+            adjustments.append(-0.05)
+        elif text_len >= 50:
+            adjustments.append(0.02)
+
+        roles = [s.role for s in para.sentences] if para.sentences else []
+        field_count = sum(1 for r in roles if r in ("trigger", "condition", "action", "result"))
+        if field_count >= 3:
+            adjustments.append(0.05)
+        elif field_count == 1:
+            adjustments.append(-0.02)
+
+        unique_roles = set(roles)
+        if "trigger" in unique_roles and "action" in unique_roles:
+            adjustments.append(0.03)
+        if "condition" in unique_roles and "action" in unique_roles:
+            adjustments.append(0.02)
+
+        confidence = base + sum(adjustments)
+        return max(0.5, min(0.99, confidence))
+
+    def calculate_vision_confidence(
+        self,
+        page_type: str,
+        component_count: int
+    ) -> float:
+        """
+        Calculate confidence for Vision-derived functions.
+
+        Args:
+            page_type: Type of page
+            component_count: Number of UI components detected
+
+        Returns:
+            Confidence score between 0.0 and 1.0
+        """
+        base = self.SOURCE_BASE["vision"]
+
+        adjustments = []
+
+        if page_type and page_type not in ("Unknown", "Other"):
+            adjustments.append(0.05)
+        else:
+            adjustments.append(-0.05)
+
+        if component_count == 0:
+            adjustments.append(-0.10)
+        elif component_count <= 10:
+            adjustments.append(0.03)
+        elif component_count > 20:
+            adjustments.append(-0.02)
+
+        confidence = base + sum(adjustments)
+        return max(0.5, min(0.95, confidence))
+
+    def _get_base_confidence(self, source_hint: str) -> float:
+        """Get base confidence for a source hint."""
+        hint_lower = source_hint.lower()
+
+        for key, val in self.SOURCE_BASE.items():
+            if key in hint_lower:
+                return val
+
+        for alias, canonical in self.SOURCE_ALIASES.items():
+            if alias in hint_lower:
+                return self.SOURCE_BASE.get(canonical, 0.9)
+
+        return 0.9

memnex/processing/merger/conflict_resolver.py

@@ -0,0 +1,116 @@
+"""
+Detect and resolve conflicts between extracted data.
+
+Design principle (v3.2 §1.6): Field multi-value coexistence (non-authority arbitration).
+When conflicts occur, ALL values are preserved and needs_review is set to True.
+Only when user manually resolves does one value become the "final" value.
+"""
+
+from typing import List, Dict, Optional
+from dataclasses import dataclass, field
+
+
+@dataclass
+class Conflict:
+    id: str
+    type: str  # field_value, missing_field, etc.
+    severity: str  # high, medium, low
+    field: str
+    values: List[Dict]  # [{"source": ..., "content": ..., "authority": ...}]
+    resolved: bool = False
+    final_value: Optional[str] = None
+    needs_human: bool = True
+
+
+class ConflictResolver:
+    """
+    Detects and resolves conflicts in extracted data.
+
+    Follows v3.2 §1.6 design: multi-value coexistence (no authority arbitration).
+    All conflicting values are preserved, needs_human=True by default.
+    """
+
+    def detect_conflicts(self, functions: List) -> List[Conflict]:
+        """Detect conflicts between functions."""
+        conflicts = []
+        conflict_id = 1
+
+        func_map = {}
+        for func in functions:
+            key = func.name_normalized
+            if key not in func_map:
+                func_map[key] = []
+            func_map[key].append(func)
+
+        for key, funcs in func_map.items():
+            if len(funcs) < 2:
+                continue
+
+            for i in range(len(funcs)):
+                for j in range(i + 1, len(funcs)):
+                    conflict = self._compare_functions(funcs[i], funcs[j], conflict_id)
+                    if conflict:
+                        conflicts.append(conflict)
+                        conflict_id += 1
+
+        return conflicts
+
+    def _compare_functions(self, func1, func2, conflict_id: int) -> Optional[Conflict]:
+        """Compare two functions for conflicts."""
+        # Compare conditions (adapted for List[FieldValue])
+        cond1_descs = [fv.desc for fv in func1.condition] if func1.condition else []
+        cond2_descs = [fv.desc for fv in func2.condition] if func2.condition else []
+
+        if cond1_descs and cond2_descs:
+            if cond1_descs != cond2_descs:
+                auth1 = func1.source_authority or "unknown"
+                auth2 = func2.source_authority or "unknown"
+                return Conflict(
+                    id=f"conflict_{conflict_id:03d}",
+                    type="field_value",
+                    severity="medium",
+                    field="condition",
+                    values=[
+                        {"source": func1.source_paragraphs[0] if func1.source_paragraphs else "unknown",
+                         "content": ", ".join(cond1_descs),
+                         "authority": auth1},
+                        {"source": func2.source_paragraphs[0] if func2.source_paragraphs else "unknown",
+                         "content": ", ".join(cond2_descs),
+                         "authority": auth2}
+                    ],
+                    needs_human=True
+                )
+        return None
+
+    def get_all_values(self, conflict: Conflict) -> List[str]:
+        """Get all conflicting values."""
+        if not conflict.values:
+            return []
+        return [v["content"] for v in conflict.values]
+
+    def mark_for_human_review(self, conflict: Conflict, suggestion: str = None):
+        """Mark conflict for human review."""
+        conflict.needs_human = True
+        conflict.resolved = False
+
+    def apply_resolution(self, conflict: Conflict, value: str):
+        """Apply human resolution."""
+        if value not in self.get_all_values(conflict):
+            raise ValueError(f"Resolution value '{value}' not in conflict values")
+        conflict.final_value = value
+        conflict.resolved = True
+        conflict.needs_human = False
+
+    def resolve_conflicts(self, conflicts: List[Conflict]) -> tuple:
+        """Process conflicts, marking all for human review."""
+        unresolved = []
+        resolved = []
+
+        for conflict in conflicts:
+            if conflict.resolved and conflict.final_value:
+                resolved.append(conflict)
+            else:
+                conflict.needs_human = True
+                unresolved.append(conflict)
+
+        return unresolved, resolved

memnex/retrieval/__init__.py

@@ -0,0 +1 @@
+"""Search and ranking: embedding, reranking, deduplication."""