rewind-memory 0.1.0__py3-none-any.whl

rewind/__init__.py

@@ -0,0 +1,7 @@
+"""Rewind — 7-layer bio-inspired memory architecture for AI agents."""
+
+__version__ = "0.1.0"
+
+from rewind.client import RewindClient
+
+__all__ = ["RewindClient", "__version__"]

rewind/__main__.py

@@ -0,0 +1,213 @@
+"""Rewind CLI — memory stack for AI agents.
+
+Usage:
+    rewind ingest <path> [--tier free|pro|enterprise] [--db ./data/vector.db]
+    rewind search <query> [--tier free|pro|enterprise] [--limit 10]
+    rewind health [--tier free|pro|enterprise]
+    rewind stats
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+import sys
+
+
+def cmd_ingest(args: argparse.Namespace) -> None:
+    """Ingest files into the memory stack."""
+    from rewind.config import default_config
+    from rewind.ingest import IngestPipeline
+    from pathlib import Path
+
+    cfg = default_config(args.tier)
+    if args.db:
+        cfg.vector_db_path = args.db
+
+    pipeline = IngestPipeline(cfg)
+
+    path = Path(args.path)
+    if path.is_dir():
+        stats = pipeline.ingest_directory(str(path))
+    elif path.is_file():
+        stats = pipeline.ingest_file(str(path))
+    else:
+        print(f"Error: {args.path} not found", file=sys.stderr)
+        sys.exit(1)
+
+    pipeline.close()
+
+    # Print summary
+    print(f"Files:    {stats.get('files', 1)}")
+    print(f"Chunks:   {stats.get('chunks', 0)}")
+    print(f"Vectors:  {stats.get('vectors', 0)}")
+    print(f"Entities: {stats.get('entities', 0)}")
+    if "elapsed_seconds" in stats:
+        print(f"Time:     {stats['elapsed_seconds']}s")
+    if stats.get("errors"):
+        print(f"Errors:   {stats['errors']}")
+
+
+def cmd_search(args: argparse.Namespace) -> None:
+    """Search the memory stack."""
+    from rewind.client import RewindClient
+    from rewind.config import default_config
+
+    cfg = default_config(args.tier)
+    if args.db:
+        cfg.vector_db_path = args.db
+
+    client = RewindClient(cfg)
+    results = client.search(args.query, limit=args.limit)
+    client.close()
+
+    if not results:
+        print("No results found.")
+        return
+
+    if args.json:
+        print(json.dumps([{
+            "score": r.score, "layer": r.layer,
+            "path": r.path, "text": r.text[:200],
+        } for r in results], indent=2))
+    else:
+        for i, r in enumerate(results, 1):
+            print(f"\n{'─' * 60}")
+            print(f"  #{i}  score={r.score:.3f}  layer={r.layer}")
+            if r.path:
+                print(f"  path: {r.path}")
+            print(f"  {r.text[:200]}")
+
+
+def cmd_health(args: argparse.Namespace) -> None:
+    """Health check."""
+    from rewind.client import RewindClient
+    from rewind.config import default_config
+
+    cfg = default_config(args.tier)
+    if args.db:
+        cfg.vector_db_path = args.db
+
+    client = RewindClient(cfg)
+    health = client.health()
+    client.close()
+
+    print(f"Tier:         {health['tier']}")
+    print(f"Orchestrator: {'✅' if health['orchestrator'] else '❌'}")
+    print(f"Bio:          {'✅' if health['bio'] else '❌'}")
+    print(f"Healthy:      {'✅' if health['healthy'] else '❌'}")
+    print()
+    for name, status in health.get("layers", {}).items():
+        s = status.get("status", "unknown")
+        icon = "✅" if s == "ok" else "❌" if s == "error" else "⚠️"
+        extra = ""
+        if "nodes" in status:
+            extra = f" ({status['nodes']} nodes, {status.get('edges', 0)} edges)"
+        elif "total_chunks" in status:
+            extra = f" ({status['total_chunks']} chunks)"
+        elif "backend" in status:
+            extra = f" ({status['backend']})"
+        print(f"  {icon} {name}: {s}{extra}")
+
+
+def cmd_migrate(args: argparse.Namespace) -> None:
+    """Migrate SQLite backends to Neo4j/Qdrant."""
+    from rewind.migrate import Migrator
+
+    db = args.db or "./data/vector.db"
+    migrator = Migrator(sqlite_db=db)
+
+    if args.subcmd == "status":
+        import json
+        print(json.dumps(migrator.status(), indent=2))
+        return
+
+    if args.subcmd == "graph":
+        result = migrator.migrate_graph_to_neo4j(
+            uri=args.uri or "bolt://localhost:7687",
+            user=args.user or "neo4j",
+            password=args.password or "",
+            dry_run=args.dry_run,
+        )
+    elif args.subcmd == "vectors":
+        result = migrator.migrate_vectors_to_qdrant(
+            qdrant_url=args.uri or "http://localhost:6333",
+            collection=args.collection or "rewind_chunks",
+            dry_run=args.dry_run,
+        )
+    else:
+        print("Usage: rewind migrate {status|graph|vectors}", file=sys.stderr)
+        sys.exit(1)
+
+    if "error" in result:
+        print(f"Error: {result['error']}", file=sys.stderr)
+        sys.exit(1)
+
+    import json
+    print(json.dumps(result, indent=2))
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        prog="rewind",
+        description="Rewind — bio-inspired memory stack for AI agents",
+    )
+    parser.add_argument(
+        "-v", "--verbose", action="store_true",
+        help="Enable debug logging",
+    )
+
+    sub = parser.add_subparsers(dest="command")
+
+    # ingest
+    p_ingest = sub.add_parser("ingest", help="Ingest files into memory")
+    p_ingest.add_argument("path", help="File or directory to ingest")
+    p_ingest.add_argument("--tier", default="free", choices=["free", "pro", "enterprise"])
+    p_ingest.add_argument("--db", default=None, help="Path to vector DB")
+
+    # search
+    p_search = sub.add_parser("search", help="Search memory")
+    p_search.add_argument("query", help="Search query")
+    p_search.add_argument("--tier", default="free", choices=["free", "pro", "enterprise"])
+    p_search.add_argument("--db", default=None, help="Path to vector DB")
+    p_search.add_argument("--limit", type=int, default=10)
+    p_search.add_argument("--json", action="store_true", help="Output JSON")
+
+    # health
+    p_health = sub.add_parser("health", help="Health check")
+    p_health.add_argument("--tier", default="free", choices=["free", "pro", "enterprise"])
+    p_health.add_argument("--db", default=None, help="Path to vector DB")
+
+    # migrate
+    p_migrate = sub.add_parser("migrate", help="Migrate backends (Pro)")
+    p_migrate.add_argument("subcmd", choices=["status", "graph", "vectors"],
+                           help="Migration target")
+    p_migrate.add_argument("--db", default=None, help="Path to SQLite DB")
+    p_migrate.add_argument("--uri", default=None, help="Target URI (bolt:// or http://)")
+    p_migrate.add_argument("--user", default="neo4j", help="Neo4j username")
+    p_migrate.add_argument("--password", default="", help="Neo4j password")
+    p_migrate.add_argument("--collection", default="rewind_chunks", help="Qdrant collection")
+    p_migrate.add_argument("--dry-run", action="store_true", help="Report only, don't write")
+
+    args = parser.parse_args()
+
+    if args.verbose:
+        logging.basicConfig(level=logging.DEBUG, format="%(levelname)s %(name)s: %(message)s")
+    else:
+        logging.basicConfig(level=logging.WARNING)
+
+    if args.command == "ingest":
+        cmd_ingest(args)
+    elif args.command == "search":
+        cmd_search(args)
+    elif args.command == "health":
+        cmd_health(args)
+    elif args.command == "migrate":
+        cmd_migrate(args)
+    else:
+        parser.print_help()
+
+
+if __name__ == "__main__":
+    main()

rewind/client.py

@@ -0,0 +1,289 @@
+"""RewindClient — unified interface to the memory stack.
+
+Free tier: L0 (FTS5), L1 (System), L3 (SQLite Graph), L4 (Vector).
+Pro tier: extends with L5, L6, Bio, Feedback via rewind-memory-pro plugin.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Callable, Dict, List, Optional
+
+from rewind.config import RewindConfig, default_config
+
+log = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Result type
+# ---------------------------------------------------------------------------
+
+@dataclass
+class Result:
+    path: str = ""
+    text: str = ""
+    score: float = 0.0
+    layer: str = ""
+    source: str = ""
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+
+# ---------------------------------------------------------------------------
+# Plugin registry — Pro layers register themselves here
+# ---------------------------------------------------------------------------
+
+_PRO_PLUGIN: Optional[Any] = None
+
+
+def register_pro_plugin(plugin: Any) -> None:
+    """Called by rewind-memory-pro on import to extend the client."""
+    global _PRO_PLUGIN
+    _PRO_PLUGIN = plugin
+    log.info("Rewind Pro plugin registered")
+
+
+# ---------------------------------------------------------------------------
+# Client
+# ---------------------------------------------------------------------------
+
+class RewindClient:
+    """Unified gateway to the memory architecture.
+
+    Usage::
+
+        from rewind.client import RewindClient
+        from rewind.config import default_config
+
+        client = RewindClient(default_config("free"))
+        results = client.search("what is Hebbian learning")
+        for r in results:
+            print(f"{r.score:.2f} [{r.layer}] {r.text[:80]}")
+    """
+
+    def __init__(
+        self,
+        config: Optional[RewindConfig] = None,
+        embedding_fn: Optional[Callable[[str], List[float]]] = None,
+        api_key: Optional[str] = None,
+    ) -> None:
+        self.config = config or default_config()
+        self._embedding_fn = embedding_fn
+        self._api_key = api_key
+        self._layers: Dict[str, Any] = {}
+        self._orchestrator = None
+        self._initialised = False
+
+    # -- lazy init --------------------------------------------------------
+
+    def _init_layers(self) -> None:
+        if self._initialised:
+            return
+
+        cfg = self.config
+        lf = cfg.layers
+
+        workspace = os.path.expanduser(cfg.workspace_path)
+        db_path = os.path.expanduser(cfg.vector_db_path)
+
+        # -- L0: FTS5/BM25 keyword search --
+        if lf.l0_sensory:
+            try:
+                from rewind.layers.l0_fts import L0FullTextSearch
+                self._layers["l0"] = L0FullTextSearch(db_path)
+                log.info("L0 (FTS5) initialised: %s", db_path)
+            except Exception as exc:
+                log.warning("L0 init failed: %s", exc)
+
+        # -- L1: System files --
+        if lf.l1_stm:
+            try:
+                from rewind.layers.l1_system import L1SystemFiles
+                self._layers["l1"] = L1SystemFiles(workspace)
+                log.info("L1 (System Files) initialised: %s", workspace)
+            except Exception as exc:
+                log.warning("L1 init failed: %s", exc)
+
+        # -- L3: Knowledge graph (SQLite) --
+        if lf.l3_graph:
+            try:
+                from rewind.layers.l3_graph_sqlite import L3GraphSQLite
+                graph_db = str(Path(db_path).parent / "graph.db")
+                self._layers["l3"] = L3GraphSQLite(graph_db)
+                log.info("L3 (Graph/SQLite) initialised: %s", graph_db)
+            except Exception as exc:
+                log.warning("L3 init failed: %s", exc)
+
+        # -- L4: Vector search (sqlite-vec) --
+        if lf.l4_workspace:
+            try:
+                from rewind.layers.l4_vector import L4Vector
+                self._layers["l4"] = L4Vector(
+                    db_path=db_path,
+                    embedding_fn=self._get_embedding_fn(),
+                )
+                log.info("L4 (Vector) initialised: %s", db_path)
+            except Exception as exc:
+                log.warning("L4 init failed: %s", exc)
+
+        # -- Pro layers (via plugin) --
+        if _PRO_PLUGIN and self._api_key:
+            try:
+                pro_layers = _PRO_PLUGIN.init_pro_layers(
+                    config=cfg,
+                    api_key=self._api_key,
+                    db_path=db_path,
+                    embedding_fn=self._get_embedding_fn(),
+                )
+                self._layers.update(pro_layers)
+                log.info("Pro layers initialised: %s", list(pro_layers.keys()))
+            except Exception as exc:
+                log.warning("Pro layer init failed: %s", exc)
+        elif any([lf.l5_comms, lf.l6_docs, lf.bio]):
+            log.warning(
+                "Pro features requested but rewind-memory-pro is not installed "
+                "or no API key provided. Install with: "
+                "pip install git+https://github.com/saraidefence/rewind-memory-pro.git"
+            )
+
+        # -- L2: Orchestrator --
+        if self._layers:
+            try:
+                from rewind.layers.l2_orchestrator import L2Orchestrator
+                orch_config = {f"enable_{k}": True for k in self._layers}
+                self._orchestrator = L2Orchestrator(orch_config, self._layers)
+                log.info("L2 (Orchestrator) initialised with layers: %s",
+                         list(self._layers.keys()))
+            except Exception as exc:
+                log.warning("L2 orchestrator init failed: %s", exc)
+
+        self._initialised = True
+
+    def _get_embedding_fn(self) -> Optional[Callable[[str], List[float]]]:
+        """Return the embedding function."""
+        if self._embedding_fn:
+            return self._embedding_fn
+
+        cfg = self.config.embedding
+        if cfg.provider == "ollama":
+            return self._make_ollama_embedder(cfg.url, cfg.model)
+
+        # Pro embedding providers handled by plugin
+        if _PRO_PLUGIN and self._api_key:
+            fn = _PRO_PLUGIN.get_embedding_fn(cfg, self._api_key)
+            if fn:
+                return fn
+
+        return self._make_ollama_embedder(cfg.url, cfg.model)
+
+    @staticmethod
+    def _make_ollama_embedder(url: str, model: str) -> Callable[[str], List[float]]:
+        def embed(text: str) -> List[float]:
+            try:
+                import httpx
+                resp = httpx.post(
+                    f"{url}/api/embed",
+                    json={"model": model, "input": text},
+                    timeout=10,
+                )
+                resp.raise_for_status()
+                data = resp.json()
+                return data["embeddings"][0] if "embeddings" in data else data["embedding"]
+            except Exception:
+                return []
+        return embed
+
+    # -- public API -------------------------------------------------------
+
+    def search(
+        self,
+        query: str,
+        limit: int = 16,
+        namespace: Optional[str] = None,
+    ) -> List[Result]:
+        """Search across all enabled layers."""
+        self._init_layers()
+
+        if self._orchestrator:
+            raw_results = self._orchestrator.search(
+                query, limit=limit, namespace=namespace,
+            )
+            return [
+                Result(
+                    path=r.get("path", ""),
+                    text=r.get("text", ""),
+                    score=r.get("score", 0.0),
+                    layer=r.get("layer", ""),
+                    source=r.get("source", ""),
+                    metadata={k: v for k, v in r.items()
+                              if k not in ("path", "text", "score", "layer", "source")},
+                )
+                for r in raw_results
+            ][:limit]
+
+        # Fallback: sequential search
+        results: List[Result] = []
+        for name, layer in self._layers.items():
+            try:
+                layer_results = layer.search(query, limit=limit, namespace=namespace)
+                if isinstance(layer_results, list):
+                    for r in layer_results:
+                        if isinstance(r, dict):
+                            results.append(Result(
+                                path=r.get("path", ""),
+                                text=r.get("text", ""),
+                                score=r.get("score", 0.0),
+                                layer=r.get("layer", name),
+                                source=r.get("source", ""),
+                            ))
+            except Exception as exc:
+                log.warning("Search failed on %s: %s", name, exc)
+
+        results.sort(key=lambda r: r.score, reverse=True)
+        return results[:limit]
+
+    def ingest(self, path: str, arena: str = "general") -> dict:
+        """Ingest a document through L0 sensory buffer."""
+        self._init_layers()
+        l0 = self._layers.get("l0")
+        if l0 and hasattr(l0, "ingest"):
+            return l0.ingest(path, arena)
+        return {"status": "error", "detail": "L0 sensory layer not available"}
+
+    def health(self) -> dict:
+        """Health check across all enabled layers."""
+        self._init_layers()
+        report: Dict[str, Any] = {
+            "tier": self.config.tier,
+            "layers": {},
+            "orchestrator": self._orchestrator is not None,
+            "pro_installed": _PRO_PLUGIN is not None,
+        }
+        for key, layer in self._layers.items():
+            try:
+                if hasattr(layer, "health"):
+                    report["layers"][key] = layer.health()
+                else:
+                    report["layers"][key] = {"status": "ok"}
+            except Exception as exc:
+                report["layers"][key] = {"status": "error", "detail": str(exc)}
+
+        report["healthy"] = all(
+            v.get("status") in ("ok", "degraded")
+            for v in report["layers"].values()
+        )
+        return report
+
+    def close(self) -> None:
+        """Shut down all layer connections."""
+        for name, layer in self._layers.items():
+            try:
+                if hasattr(layer, "close"):
+                    layer.close()
+            except Exception as exc:
+                log.warning("Close failed on %s: %s", name, exc)
+        self._layers.clear()
+        self._orchestrator = None
+        self._initialised = False

rewind/config.py

@@ -0,0 +1,176 @@
+"""Rewind configuration — YAML-driven, tier-aware, no hardcoded paths."""
+
+from __future__ import annotations
+
+import json
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+import yaml
+
+
+# ---------------------------------------------------------------------------
+# Sub-configs
+# ---------------------------------------------------------------------------
+
+@dataclass
+class EmbeddingConfig:
+    provider: str = "ollama"          # ollama | nv_embed | openai
+    url: str = "http://localhost:11434"
+    model: str = "nomic-embed-text"
+    dimension: int = 768
+
+
+@dataclass
+class Neo4jConfig:
+    uri: str = "bolt://localhost:7687"
+    user: str = "neo4j"
+    password: str = ""
+    secrets_file: Optional[str] = None   # path to JSON with {"password": "..."}
+
+    def resolve_password(self) -> str:
+        """Return password, loading from secrets file if configured."""
+        if self.secrets_file and not self.password:
+            p = Path(self.secrets_file).expanduser()
+            if p.exists():
+                data = json.loads(p.read_text())
+                self.password = data.get("password", "")
+        return self.password
+
+
+@dataclass
+class QdrantConfig:
+    url: str = "http://localhost:6333"
+
+
+@dataclass
+class LayerFlags:
+    l0_sensory: bool = True       # always on — FTS5/BM25 keyword search
+    l1_stm: bool = True           # short-term memory (sqlite-vec)
+    l2_orchestrator: bool = True  # search router — implicitly on when any store is
+    l3_graph: bool = True         # knowledge graph (SQLite default, Neo4j optional)
+    l4_workspace: bool = True     # file-system workspace memory (sqlite-vec)
+    l5_comms: bool = False        # communications store (sqlite-vec default, Qdrant optional)
+    l6_docs: bool = False         # document store
+    bio: bool = False             # bio-inspired lifecycle (decay, consolidation, pruning)
+
+
+# ---------------------------------------------------------------------------
+# Main config
+# ---------------------------------------------------------------------------
+
+@dataclass
+class RewindConfig:
+    layers: LayerFlags = field(default_factory=LayerFlags)
+    embedding: EmbeddingConfig = field(default_factory=EmbeddingConfig)
+    neo4j: Neo4jConfig = field(default_factory=Neo4jConfig)
+    qdrant: QdrantConfig = field(default_factory=QdrantConfig)
+    vector_db_path: str = "./data/vector.db"
+    workspace_path: str = "./workspace"
+    tier: str = "free"
+
+    # Arbitrary extra keys from YAML
+    extra: Dict[str, Any] = field(default_factory=dict)
+
+
+# ---------------------------------------------------------------------------
+# Tier presets
+# ---------------------------------------------------------------------------
+
+_TIER_LAYERS: Dict[str, Dict[str, bool]] = {
+    "free": dict(
+        l0_sensory=True, l1_stm=True, l2_orchestrator=True,
+        l3_graph=True, l4_workspace=True, l5_comms=False,
+        l6_docs=False, bio=False,
+    ),
+    "pro": dict(
+        l0_sensory=True, l1_stm=True, l2_orchestrator=True,
+        l3_graph=True, l4_workspace=True, l5_comms=True,
+        l6_docs=True, bio=False,
+    ),
+    "enterprise": dict(
+        l0_sensory=True, l1_stm=True, l2_orchestrator=True,
+        l3_graph=True, l4_workspace=True, l5_comms=True,
+        l6_docs=True, bio=True,
+    ),
+}
+
+_TIER_EMBEDDING: Dict[str, Dict[str, Any]] = {
+    "free": dict(provider="ollama", url="http://localhost:11434",
+                 model="nomic-embed-text", dimension=768),
+    "pro": dict(provider="nv_embed", url="http://localhost:8080",
+                model="NV-Embed-v2", dimension=4096),
+    "enterprise": dict(provider="nv_embed", url="http://localhost:8080",
+                       model="NV-Embed-v2", dimension=4096),
+}
+
+
+def detect_tier(layers: LayerFlags) -> str:
+    """Infer tier string from enabled layers."""
+    if layers.bio:
+        return "enterprise"
+    if layers.l5_comms and layers.l6_docs:
+        return "pro"
+    return "free"
+
+
+# ---------------------------------------------------------------------------
+# Builders
+# ---------------------------------------------------------------------------
+
+def _build_sub(cls, raw: dict | None):
+    """Instantiate a dataclass from a dict, ignoring unknown keys."""
+    if not raw:
+        return cls()
+    valid = {f.name for f in cls.__dataclass_fields__.values()}
+    return cls(**{k: v for k, v in raw.items() if k in valid})
+
+
+def load_config(path: str) -> RewindConfig:
+    """Load an RewindConfig from a YAML file."""
+    p = Path(path).expanduser()
+    with p.open() as f:
+        raw: dict = yaml.safe_load(f) or {}
+
+    layers = _build_sub(LayerFlags, raw.get("layers"))
+    embedding = _build_sub(EmbeddingConfig, raw.get("embedding"))
+    neo4j = _build_sub(Neo4jConfig, raw.get("neo4j"))
+    qdrant = _build_sub(QdrantConfig, raw.get("qdrant"))
+
+    # Resolve neo4j password from env or secrets file
+    if not neo4j.password:
+        neo4j.password = os.environ.get("REWIND_NEO4J_PASSWORD", "")
+    neo4j.resolve_password()
+
+    known = {"layers", "embedding", "neo4j", "qdrant",
+             "vector_db_path", "workspace_path", "tier"}
+    extra = {k: v for k, v in raw.items() if k not in known}
+
+    cfg = RewindConfig(
+        layers=layers,
+        embedding=embedding,
+        neo4j=neo4j,
+        qdrant=qdrant,
+        vector_db_path=raw.get("vector_db_path", "./data/vector.db"),
+        workspace_path=raw.get("workspace_path", "./workspace"),
+        tier=raw.get("tier", detect_tier(layers)),
+        extra=extra,
+    )
+    return cfg
+
+
+def default_config(tier: str = "free") -> RewindConfig:
+    """Return a sensible default config for the given tier."""
+    tier = tier.lower()
+    if tier not in _TIER_LAYERS:
+        raise ValueError(f"Unknown tier {tier!r}; choose free/pro/enterprise")
+
+    return RewindConfig(
+        layers=LayerFlags(**_TIER_LAYERS[tier]),
+        embedding=EmbeddingConfig(**_TIER_EMBEDDING[tier]),
+        neo4j=Neo4jConfig(),
+        qdrant=QdrantConfig(),
+        tier=tier,
+    )

rewind/embedding/__init__.py

@@ -0,0 +1,13 @@
+"""Embedding providers."""
+
+from .base import EmbeddingProvider
+from .nv_embed import NVEmbedProvider
+from .ollama import OllamaProvider
+from .openai import OpenAIProvider
+
+__all__ = [
+    "EmbeddingProvider",
+    "NVEmbedProvider",
+    "OllamaProvider",
+    "OpenAIProvider",
+]