crabpath 1.0.0__py3-none-any.whl

crabpath/__init__.py

@@ -0,0 +1,191 @@
+"""
+Most users only need: from crabpath import Graph, MemoryController
+
+🦀 CrabPath: The Graph is the Prompt.
+
+LLM-guided memory traversal with learned pointer weights
+and corrected policy gradients.
+
+CLI:
+  python -m crabpath
+  crabpath  # via console_scripts entry point
+
+Paper: https://jonathangu.com/crabpath/
+"""
+
+__version__ = "1.0.0"
+
+from .adapter import CrabPathAgent, OpenClawCrabPathAdapter
+from .autotune import (
+    DEFAULTS,
+    HEALTH_TARGETS,
+    Adjustment,
+    GraphHealth,
+    autotune,
+    measure_health,
+    suggest_config,
+)
+from .controller import ControllerConfig, MemoryController, QueryResult
+from .embeddings import (
+    EmbeddingIndex,
+    auto_embed,
+    cohere_embed,
+    gemini_embed,
+    ollama_embed,
+    openai_embed,
+)
+from .feedback import (
+    auto_feedback,
+    auto_outcome,
+    detect_correction,
+    map_correction_to_snapshot,
+    score_retrieval,
+)
+from .graph import (
+    ConsolidationConfig,
+    ConsolidationResult,
+    Edge,
+    Graph,
+    Node,
+    consolidate,
+    prune_orphan_nodes,
+    prune_probationary,
+    prune_weak_edges,
+    should_split,
+)
+from .inhibition import (
+    InhibitionConfig,
+    apply_correction,
+    get_inhibitory_edges,
+    inhibition_stats,
+    is_inhibited,
+    score_with_inhibition,
+)
+from .learning import LearningConfig, LearningResult, RewardSignal, make_learning_step
+from .legacy.activation import Firing, activate, learn
+from .migrate import MigrateConfig, gather_files, migrate, parse_session_logs
+from .mitosis import (
+    BLOCKED_QUERIES,
+    MergeResult,
+    MitosisConfig,
+    MitosisState,
+    NeurogenesisConfig,
+    NeurogenesisResult,
+    NoveltyResult,
+    SplitResult,
+    assess_novelty,
+    bootstrap_workspace,
+    connect_new_node,
+    create_node,
+    deterministic_auto_id,
+    find_co_firing_families,
+    merge_nodes,
+    mitosis_maintenance,
+    should_create_node,
+    should_merge,
+    split_node,
+    split_with_llm,
+)
+from .shadow_logger import ShadowLog
+from .synaptogenesis import (
+    ProtoEdge,
+    SynaptogenesisConfig,
+    SynaptogenesisState,
+    classify_tier,
+    decay_proto_edges,
+    edge_tier_stats,
+    record_cofiring,
+    record_correction,
+    record_skips,
+)
+
+__all__ = [
+    # --- Core (start here) ---
+    "Graph",
+    "Node",
+    "Edge",
+    "MemoryController",
+    "QueryResult",
+    "ControllerConfig",
+
+    # --- Embeddings ---
+    "EmbeddingIndex",
+    "auto_embed",
+    "openai_embed",
+    "gemini_embed",
+    "cohere_embed",
+    "ollama_embed",
+
+    # --- Learning ---
+    "activate",
+    "learn",
+    "Firing",
+    "LearningConfig",
+    "LearningResult",
+    "RewardSignal",
+    "make_learning_step",
+
+    # --- Advanced (internals) ---
+    "ConsolidationConfig",
+    "ConsolidationResult",
+    "prune_orphan_nodes",
+    "prune_weak_edges",
+    "prune_probationary",
+    "should_split",
+    "consolidate",
+    "should_merge",
+    "CrabPathAgent",
+    "OpenClawCrabPathAdapter",
+    "migrate",
+    "MigrateConfig",
+    "gather_files",
+    "parse_session_logs",
+    "auto_feedback",
+    "auto_outcome",
+    "detect_correction",
+    "score_retrieval",
+    "map_correction_to_snapshot",
+    "DEFAULTS",
+    "Adjustment",
+    "GraphHealth",
+    "HEALTH_TARGETS",
+    "autotune",
+    "measure_health",
+    "suggest_config",
+    "InhibitionConfig",
+    "apply_correction",
+    "score_with_inhibition",
+    "is_inhibited",
+    "get_inhibitory_edges",
+    "inhibition_stats",
+    "MitosisConfig",
+    "MitosisState",
+    "SplitResult",
+    "MergeResult",
+    "NeurogenesisConfig",
+    "NeurogenesisResult",
+    "NoveltyResult",
+    "assess_novelty",
+    "connect_new_node",
+    "deterministic_auto_id",
+    "split_node",
+    "split_with_llm",
+    "should_merge",
+    "create_node",
+    "should_create_node",
+    "find_co_firing_families",
+    "merge_nodes",
+    "bootstrap_workspace",
+    "mitosis_maintenance",
+    "SynaptogenesisConfig",
+    "SynaptogenesisState",
+    "ProtoEdge",
+    "record_cofiring",
+    "record_skips",
+    "record_correction",
+    "classify_tier",
+    "decay_proto_edges",
+    "edge_tier_stats",
+    "ShadowLog",
+    "BLOCKED_QUERIES",
+]

crabpath/__main__.py

@@ -0,0 +1,7 @@
+"""Package entrypoint for `python -m crabpath`."""
+
+from .cli import main
+import sys
+
+if __name__ == "__main__":
+    sys.exit(main())

crabpath/_io.py

@@ -0,0 +1,257 @@
+"""Shared IO helpers for CLI and MCP entrypoints."""
+
+from __future__ import annotations
+
+import json
+import time
+from pathlib import Path
+from typing import Any, Callable
+
+from ._structural_utils import count_cross_file_edges
+from .autotune import HEALTH_TARGETS
+from .embeddings import EmbeddingIndex
+from .graph import Graph
+from .legacy.activation import Firing, activate
+from .mitosis import MitosisState
+from .synaptogenesis import edge_tier_stats
+
+
+def load_graph(path: str) -> Graph:
+    """Load a graph with stable CLI/MCP error messages."""
+    file_path = Path(path)
+    if not file_path.exists():
+        raise FileNotFoundError(f"graph file not found: {path}")
+
+    try:
+        return Graph.load(path)
+    except (OSError, json.JSONDecodeError, KeyError, TypeError, ValueError) as exc:
+        raise ValueError(f"failed to load graph: {path}: {exc}") from exc
+
+
+def load_index(path: str) -> EmbeddingIndex:
+    """Load embedding index from disk, or return empty if missing."""
+    file_path = Path(path)
+    if not file_path.exists():
+        return EmbeddingIndex()
+
+    try:
+        return EmbeddingIndex.load(path)
+    except (OSError, json.JSONDecodeError, KeyError, TypeError) as exc:
+        raise ValueError(f"failed to load index: {path}: {exc}") from exc
+
+
+def load_query_stats(path: str | None) -> dict[str, Any]:
+    if path is None:
+        return {}
+
+    file_path = Path(path)
+    if not file_path.exists():
+        raise FileNotFoundError(f"query-stats file not found: {path}")
+
+    try:
+        raw = file_path.read_text(encoding="utf-8")
+        stats = json.loads(raw)
+    except (OSError, json.JSONDecodeError) as exc:
+        raise ValueError(f"failed to load query-stats: {path}: {exc}") from exc
+
+    if not isinstance(stats, dict):
+        raise ValueError(f"query-stats must be a JSON object: {path}")
+
+    return stats
+
+
+def load_mitosis_state(path: str | None) -> MitosisState:
+    if path is None:
+        return MitosisState()
+
+    file_path = Path(path)
+    if not file_path.exists():
+        raise FileNotFoundError(f"mitosis-state file not found: {path}")
+
+    try:
+        raw = file_path.read_text(encoding="utf-8")
+        state_data = json.loads(raw)
+    except (OSError, json.JSONDecodeError) as exc:
+        raise ValueError(f"failed to load mitosis-state: {path}: {exc}") from exc
+
+    if not isinstance(state_data, dict):
+        raise ValueError(f"mitosis-state must be a JSON object: {path}")
+
+    return MitosisState(
+        families=state_data.get("families", {}),
+        generations=state_data.get("generations", {}),
+        chunk_to_parent=state_data.get("chunk_to_parent", {}),
+    )
+
+
+def load_snapshot_rows(path: Path) -> list[dict[str, Any]]:
+    if not path.exists():
+        return []
+
+    rows: list[dict[str, Any]] = []
+    with path.open(encoding="utf-8") as stream:
+        for raw in stream:
+            text = raw.strip()
+            if not text:
+                continue
+            try:
+                row = json.loads(text)
+            except json.JSONDecodeError as exc:
+                raise ValueError(f"invalid JSON line in snapshots file: {path}: {exc}") from exc
+            if not isinstance(row, dict):
+                raise ValueError(f"invalid snapshot row in snapshots file: {path}")
+            rows.append(row)
+
+    return rows
+
+
+def split_csv(value: str) -> list[str]:
+    ids = [item.strip() for item in value.split(",") if item.strip()]
+    if not ids:
+        raise ValueError("fired-ids must contain at least one id")
+    return ids
+
+
+def graph_stats(graph: Graph) -> dict[str, Any]:
+    """Compute canonical graph summary statistics."""
+    edges = graph.edges()
+    avg_weight = sum(edge.weight for edge in edges) / len(edges) if edges else 0.0
+
+    degree: dict[str, int] = {}
+    for edge in edges:
+        degree[edge.source] = degree.get(edge.source, 0) + 1
+        degree[edge.target] = degree.get(edge.target, 0) + 1
+    top = sorted(degree.items(), key=lambda item: (-item[1], item[0]))[:5]
+
+    return {
+        "nodes": graph.node_count,
+        "edges": graph.edge_count,
+        "avg_weight": avg_weight,
+        "top_hubs": [node_id for node_id, _ in top],
+    }
+
+
+def keyword_seed(graph: Graph, query_text: str) -> dict[str, float]:
+    if not query_text:
+        return {}
+
+    needles = {token.strip().lower() for token in query_text.split() if token.strip()}
+    seeds: dict[str, float] = {}
+    for node in graph.nodes():
+        haystack = f"{node.id} {node.content}".lower()
+        score = 0.0
+        for needle in needles:
+            if needle in haystack:
+                score += 1.0
+        if score:
+            seeds[node.id] = score
+
+    return seeds
+
+
+def run_query(
+    graph: Graph,
+    index: EmbeddingIndex,
+    query_text: str,
+    *,
+    top_k: int,
+    embed_fn: Callable[[list[str]], list[list[float]]] | None = None,
+) -> Firing:
+    seeds: dict[str, float] = {}
+    if embed_fn is not None and index.vectors:
+        try:
+            seeds = index.seed(query_text, embed_fn=embed_fn, top_k=top_k)
+        except (TypeError, ValueError):
+            seeds = {}
+
+    if not seeds:
+        seeds = keyword_seed(graph, query_text)
+
+    return activate(
+        graph,
+        seeds,
+        max_steps=3,
+        decay=0.1,
+        top_k=top_k,
+        reset=False,
+    )
+
+
+def build_firing(graph: Graph, fired_ids: list[str]) -> Firing:
+    if not fired_ids:
+        raise ValueError("fired-ids must contain at least one id")
+
+    nodes: list[tuple[Any, float]] = []
+    fired_at: dict[str, int] = {}
+    for idx, node_id in enumerate(fired_ids):
+        node = graph.get_node(node_id)
+        if node is None:
+            raise ValueError(f"unknown node id: {node_id}")
+        nodes.append((node, 1.0))
+        fired_at[node_id] = idx
+
+    return Firing(fired=nodes, inhibited=[], fired_at=fired_at)
+
+
+def build_snapshot(graph: Graph) -> dict[str, Any]:
+    return {
+        "timestamp": time.time(),
+        "nodes": graph.node_count,
+        "edges": graph.edge_count,
+        "tier_counts": edge_tier_stats(graph),
+        "cross_file_edges": count_cross_file_edges(graph),
+    }
+
+
+def health_metric_available(metric: str, has_query_stats: bool) -> bool:
+    if metric in {
+        "avg_nodes_fired_per_query",
+        "context_compression",
+        "proto_promotion_rate",
+        "reconvergence_rate",
+    }:
+        return has_query_stats
+    return True
+
+
+def health_metric_status(
+    value: float | None,
+    target: tuple[float | None, float | None],
+    available: bool,
+) -> str:
+    if not available or value is None:
+        return "warn"
+
+    min_v, max_v = target
+    if min_v is not None and value < min_v:
+        return "low"
+    if max_v is not None and value > max_v:
+        return "high"
+    return "ok"
+
+
+def build_health_rows(
+    health: Any,
+    has_query_stats: bool,
+) -> list[dict[str, Any]]:
+    rows: list[dict[str, Any]] = []
+    for metric, target in HEALTH_TARGETS.items():
+        available = health_metric_available(metric, has_query_stats)
+        raw_value = getattr(health, metric, None)
+        value = raw_value if available else None
+        value_num = raw_value if isinstance(raw_value, (int, float)) else None
+        status = health_metric_status(
+            float(value_num) if value_num is not None else None,
+            target,
+            available,
+        )
+        rows.append(
+            {
+                "metric": metric,
+                "value": value if available else None,
+                "target_range": target,
+                "status": status,
+            }
+        )
+
+    return rows

crabpath/_structural_utils.py

@@ -0,0 +1,127 @@
+"""Internal structural helpers shared across modules."""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import asdict, dataclass, replace
+from typing import Any, TypeVar
+
+
+@dataclass
+class ConfigBase:
+    """Small helper base for config-style dataclasses."""
+
+    def with_updates(self: "T", **updates: Any) -> "T":
+        """Return a copy with updated fields."""
+        return replace(self, **updates)
+
+    def as_dict(self) -> dict[str, Any]:
+        """Serialize dataclass fields as a plain dictionary."""
+        return asdict(self)
+
+
+class JSONStateMixin:
+    """Small persistence helpers for JSON-backed state."""
+
+    @staticmethod
+    def _load_json_file(path: str, default: Any) -> Any:
+        try:
+            with open(path, encoding="utf-8") as f:
+                return json.load(f)
+        except (FileNotFoundError, json.JSONDecodeError):
+            return default
+        except (TypeError, OSError):
+            return default
+
+    @staticmethod
+    def _write_json_file(path: str, payload: Any, *, sort_keys: bool = False) -> None:
+        with open(path, "w", encoding="utf-8") as f:
+            json.dump(payload, f, indent=2, sort_keys=sort_keys)
+
+
+def parse_markdown_json(raw: str, *, require_object: bool = False) -> Any:
+    """Parse JSON payloads, stripping markdown JSON fences when present."""
+    if not isinstance(raw, str):
+        raise TypeError("raw model output must be a string")
+
+    cleaned = raw.strip()
+    if not cleaned:
+        raise ValueError("empty model output")
+
+    if cleaned.startswith("```"):
+        lines = cleaned.splitlines()
+        if len(lines) >= 2 and lines[0].startswith("```"):
+            if lines[-1].strip().endswith("```"):
+                lines = lines[1:-1]
+            cleaned = "\n".join(lines).strip()
+
+    payload = json.loads(cleaned)
+    if require_object and not isinstance(payload, dict):
+        raise TypeError("parsed JSON payload must be an object")
+    return payload
+
+
+def split_fallback_sections(
+    content: str,
+    *,
+    min_header_chars: int = 1,
+    min_paragraph_chars: int = 1,
+    merge_short_paragraphs: int = 0,
+) -> list[str]:
+    """Structural fallback split with heading-first decomposition."""
+    parts = [p.strip() for p in re.split(r"\n(?=## )", content) if p.strip()]
+    parts = [p for p in parts if len(p) >= max(1, min_header_chars)]
+    if len(parts) >= 2:
+        return parts
+
+    parts = [p.strip() for p in content.split("\n\n") if p.strip()]
+    parts = [p for p in parts if len(p) >= max(1, min_paragraph_chars)]
+    if len(parts) >= 2:
+        if merge_short_paragraphs > 0:
+            merged: list[str] = [parts[0]]
+            for p in parts[1:]:
+                if len(merged[-1]) < merge_short_paragraphs:
+                    merged[-1] = f"{merged[-1]}\n\n{p}"
+                else:
+                    merged.append(p)
+            if len(merged) >= 2:
+                return merged
+        return parts
+
+    return [content]
+
+
+def node_file_id(node_id: Any) -> str:
+    """Normalize node ids to a coarse file-level identifier."""
+    return str(node_id).split("::", 1)[0]
+
+
+def count_cross_file_edges(graph: Any) -> int:
+    """Count edges that cross file boundaries in node ids."""
+    if getattr(graph, "node_count", 0) <= 1:
+        return 0
+
+    return sum(
+        1
+        for edge in graph.edges()
+        if node_file_id(edge.source) != node_file_id(edge.target)
+    )
+
+
+def classify_edge_tier(
+    weight: float,
+    *,
+    dormant_threshold: float = 0.3,
+    reflex_threshold: float = 0.8,
+) -> str:
+    """Map an edge weight to a routing tier."""
+    if weight >= reflex_threshold:
+        return "reflex"
+    if weight >= dormant_threshold:
+        return "habitual"
+    return "dormant"
+
+
+T = TypeVar("T", bound=ConfigBase)
+