smartmemory-core 0.5.0__py3-none-any.whl

scripts/add_license_headers.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+"""
+Script to add AGPL v3 license headers with dual licensing notice to all Python files.
+Run this from the project root directory.
+"""
+
+import os
+from typing import List
+
+LICENSE_HEADER = '''# Copyright (C) 2025 SmartMemory
+# 
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+# 
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# For commercial licensing options, please contact: help@smartmemory.ai
+# Commercial licenses are available for organizations that wish to use
+# this software in proprietary applications without the AGPL restrictions.
+
+'''
+
+def find_python_files(directory: str) -> List[str]:
+    """Find all Python files in directory and subdirectories."""
+    python_files = []
+    for root, dirs, files in os.walk(directory):
+        # Skip hidden directories and common build/cache directories
+        dirs[:] = [d for d in dirs if not d.startswith('.') and d not in ['__pycache__', 'build', 'dist']]
+        
+        for file in files:
+            if file.endswith('.py'):
+                python_files.append(os.path.join(root, file))
+    
+    return python_files
+
+def has_license_header(file_path: str) -> bool:
+    """Check if file already has a license header."""
+    try:
+        with open(file_path, 'r', encoding='utf-8') as f:
+            content = f.read(1000)  # Check first 1000 characters
+            return 'Copyright (C) 2025 SmartMemory' in content or 'GNU Affero General Public License' in content
+    except Exception:
+        return False
+
+def add_license_header(file_path: str) -> bool:
+    """Add license header to Python file."""
+    try:
+        with open(file_path, 'r', encoding='utf-8') as f:
+            original_content = f.read()
+        
+        # Handle shebang line
+        lines = original_content.split('\n')
+        if lines and lines[0].startswith('#!'):
+            # Keep shebang, add license after it
+            shebang = lines[0] + '\n'
+            rest_content = '\n'.join(lines[1:])
+            new_content = shebang + LICENSE_HEADER + rest_content
+        else:
+            # Add license at the beginning
+            new_content = LICENSE_HEADER + original_content
+        
+        # Write back the file
+        with open(file_path, 'w', encoding='utf-8') as f:
+            f.write(new_content)
+        
+        return True
+    except Exception as e:
+        print(f"Error processing {file_path}: {e}")
+        return False
+
+def main():
+    """Main function to process all Python files."""
+    project_root = os.path.dirname(os.path.abspath(__file__))
+    
+    # Find all Python files
+    python_files = find_python_files(project_root)
+    
+    # Filter out this script itself and other utility scripts you might not want to modify
+    exclude_files = [
+        os.path.join(project_root, 'add_license_headers.py'),
+        # Add other files to exclude here if needed
+    ]
+    
+    python_files = [f for f in python_files if f not in exclude_files]
+    
+    print(f"Found {len(python_files)} Python files")
+    
+    processed = 0
+    skipped = 0
+    errors = 0
+    
+    for file_path in python_files:
+        relative_path = os.path.relpath(file_path, project_root)
+        
+        if has_license_header(file_path):
+            print(f"SKIP: {relative_path} (already has license header)")
+            skipped += 1
+            continue
+        
+        if add_license_header(file_path):
+            print(f"ADD:  {relative_path}")
+            processed += 1
+        else:
+            print(f"ERR:  {relative_path}")
+            errors += 1
+    
+    print("\nSummary:")
+    print(f"  Processed: {processed}")
+    print(f"  Skipped: {skipped}")
+    print(f"  Errors: {errors}")
+    print(f"  Total files: {len(python_files)}")
+
+if __name__ == "__main__":
+    main()

smartmemory/__init__.py

@@ -0,0 +1,29 @@
+"""
+SmartMemory: Multi-layered AI memory system with graph databases, vector stores, and
+intelligent processing pipelines for context-aware AI applications. Provides semantic,
+episodic, procedural, and working memory types with advanced relationship modeling and
+storage and retrieval for AI applications.
+"""
+
+from smartmemory.__version__ import __version__, __version_info__
+
+__author__ = "SmartMemory Team"
+
+__all__ = [
+    "SmartMemory",
+    "MemoryItem",
+    "__version__",
+    "__version_info__",
+]
+
+
+def __getattr__(name: str):
+    """Lazy-load heavy top-level exports so submodule imports don't pull the
+    full ML stack (torch, numpy, etc.) unless SmartMemory itself is used."""
+    if name == "SmartMemory":
+        from smartmemory.smart_memory import SmartMemory
+        return SmartMemory
+    if name == "MemoryItem":
+        from smartmemory.models.memory_item import MemoryItem
+        return MemoryItem
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

smartmemory/__version__.py

@@ -0,0 +1,50 @@
+"""
+SmartMemory version information.
+
+Version is read dynamically from package metadata or VERSION file.
+Single source of truth: VERSION file
+"""
+
+from pathlib import Path
+
+def _get_version() -> str:
+    """Get version from VERSION file or package metadata."""
+    try:
+        # For installed package, read from metadata
+        from importlib.metadata import version
+        return version("smartmemory-core")
+    except Exception:
+        # For development, read from VERSION file
+        try:
+            version_file = Path(__file__).parent.parent / "VERSION"
+            with open(version_file, "r", encoding="utf-8") as f:
+                return f.read().strip()
+        except FileNotFoundError:
+            # Fallback if VERSION file not found
+            return "unknown"
+
+__version__ = _get_version()
+
+# Parse version info
+try:
+    __version_info__ = tuple(int(x) for x in __version__.split("."))
+except Exception:
+    __version_info__ = (0, 1, 6)
+
+# Version history:
+# 0.3.0 - Decision Memory: Decision model, DecisionManager, DecisionQueries, DecisionExtractor, edge schemas (PRODUCED, DERIVED_FROM, SUPERSEDES, CONTRADICTS, INFLUENCES)
+# 0.2.6 - SSG integration, user signature support, documentation updates, DRY version management (VERSION file as single source of truth)
+# 0.1.14 - Tenant isolation bug fix
+# 0.1.13 - Tenant isolation bug fix
+# 0.1.12 - Tenant isolation improvements
+# 0.1.11 - Version bump and maintenance
+# 0.1.10 - Minor fix to grounder plugin, publish script improvements
+# 0.1.9 - Add dspy to requirements
+# 0.1.8 - README overhaul: fixed all code snippets to use public API, verified evolvers, added "In Progress" section, removed internal imports
+# 0.1.7 - Updated README, removed ChromaDB references, fixed PyPI deployment
+# 0.1.6 - Production PyPI deployment setup
+# 0.1.5 - Complete bi-temporal implementation: version tracking, temporal search, relationship queries, bi-temporal joins, performance optimizations
+# 0.1.4 - Bi-temporal queries: time-travel, audit trails, version history, rollback
+# 0.1.3 - Zettelkasten system with wikilink support, documentation, examples, CLI
+# 0.1.2 - ChromaDB optional, Python 3.10+ requirement, version externalized
+# 0.1.1 - Plugin system with security

smartmemory/background/extraction_worker.py

@@ -0,0 +1,186 @@
+"""Tier 2 async extraction worker — runs LLM extraction on items stored by Tier 1.
+
+Tier 1 (sync, ~4ms): spaCy + EntityRuler → stores memory, enqueues item_id.
+Tier 2 (this module, ~740ms): loads item, runs LLMSingleExtractor, diffs against
+ruler entities already stored, writes net-new entities + resolved relations,
+then updates EntityPattern nodes for the self-learning EntityRuler loop.
+"""
+from __future__ import annotations
+
+import logging
+from typing import Any, Dict, Optional, Set
+
+from smartmemory.models.memory_item import MemoryItem
+
+logger = logging.getLogger(__name__)
+
+# Pre-write quality gates for entity pattern candidates
+_MIN_CONFIDENCE = 0.8
+_MIN_NAME_LENGTH = 3
+_BLOCKLIST: Set[str] = {"it", "this", "that", "the", "a", "an", "he", "she", "they", "we"}
+
+
+def process_extract_job(
+    memory: Any,
+    payload: Dict[str, Any],
+    redis_client: Optional[Any] = None,
+) -> Dict[str, Any]:
+    """Run Tier 2 LLM extraction on a stored memory item.
+
+    Args:
+        memory: A SmartMemory (or SecureSmartMemory) instance scoped to the workspace.
+        payload: Dict from the extract queue: {item_id, workspace_id, entity_ids}.
+                 entity_ids is {name.lower(): graph_node_id} from Tier 1 StoreStage.
+        redis_client: Optional Redis connection for ruler hot-reload pub/sub.
+
+    Returns:
+        {"status": "ok"|"no_text"|"item_not_found"|"llm_failed",
+         "new_entities": int, "new_relations": int, "new_patterns": int}
+    """
+    item_id: Optional[str] = payload.get("item_id")
+    workspace_id: str = payload.get("workspace_id", "")
+    # ruler_entity_ids: {name.lower(): graph_node_id} populated by Tier 1 StoreStage
+    ruler_entity_ids: Dict[str, str] = payload.get("entity_ids") or {}
+    # Respect enable_ontology flag set by the enqueuing SmartMemory instance.
+    # When False, skip EntityPattern writes and ruler hot-reload entirely.
+    enable_ontology: bool = payload.get("enable_ontology", True)
+
+    # --- 1. Load stored item ---
+    item = memory.get(item_id)
+    if item is None:
+        logger.warning("Extract job: item_id=%s not found", item_id)
+        return {"status": "item_not_found", "new_entities": 0, "new_relations": 0, "new_patterns": 0}
+
+    content: str = ""
+    if isinstance(item, MemoryItem):
+        content = item.content or ""
+    elif isinstance(item, dict):
+        content = item.get("content", "") or ""
+
+    if not content.strip():
+        return {"status": "no_text", "new_entities": 0, "new_relations": 0, "new_patterns": 0}
+
+    # --- 2. Run LLM extraction ---
+    try:
+        from smartmemory.plugins.extractors.llm_single import LLMSingleExtractor
+
+        extractor = LLMSingleExtractor()
+        extraction = extractor.extract(content)
+    except Exception as exc:
+        logger.warning("Extract job: LLM extraction failed for item_id=%s: %s", item_id, exc)
+        return {"status": "llm_failed", "new_entities": 0, "new_relations": 0, "new_patterns": 0}
+
+    llm_entities: list[MemoryItem] = extraction.get("entities") or []
+    llm_relations: list[dict] = extraction.get("relations") or []
+
+    # --- 3. Build id remapping and filter valid relations ---
+    from smartmemory.background.id_resolver import build_sha256_to_stored, filter_valid_relations
+
+    sha256_to_stored = build_sha256_to_stored(llm_entities, ruler_entity_ids)
+
+    # --- 4. Write net-new entities (those NOT already in ruler_entity_ids) ---
+    net_new_ids: Set[str] = set()
+    new_entity_count = 0
+    ruler_names_lower = {n.lower() for n in ruler_entity_ids}
+
+    for entity in llm_entities:
+        name = (entity.metadata.get("name") or "").strip()
+        if not name or name.lower() in ruler_names_lower:
+            continue
+        # Net-new entity — write to graph via simple add()
+        try:
+            result = memory._crud.add(entity)
+            if isinstance(result, dict):
+                net_new_node_id = result.get("memory_node_id") or entity.item_id
+            else:
+                net_new_node_id = result or entity.item_id
+            if net_new_node_id:
+                net_new_ids.add(net_new_node_id)
+                # Update sha256→stored so relations can resolve this entity too
+                if entity.item_id:
+                    sha256_to_stored[entity.item_id] = net_new_node_id
+            new_entity_count += 1
+        except Exception as exc:
+            logger.warning("Extract job: failed to write net-new entity '%s': %s", name, exc)
+
+    # --- 5. Write resolved relations ---
+    resolved_relations = filter_valid_relations(llm_relations, sha256_to_stored, net_new_ids)
+    new_relation_count = 0
+    for rel in resolved_relations:
+        try:
+            memory._graph.add_edge(
+                rel["source_id"],
+                rel["target_id"],
+                edge_type=rel["relation_type"],
+                properties={},
+            )
+            new_relation_count += 1
+        except Exception as exc:
+            logger.warning(
+                "Extract job: failed to write relation %s->%s (%s): %s",
+                rel["source_id"],
+                rel["target_id"],
+                rel["relation_type"],
+                exc,
+            )
+
+    # --- 6+7. Update EntityPattern nodes + notify ruler hot-reload ---
+    # Guarded by enable_ontology: when False (e.g. SmartMemory(enable_ontology=False)),
+    # skip all ontology graph writes and Redis pub/sub entirely.
+    new_pattern_count = 0
+    if enable_ontology:
+        try:
+            from smartmemory.graph.ontology_graph import OntologyGraph
+
+            ontology_graph = OntologyGraph(workspace_id=workspace_id)
+            for entity in llm_entities:
+                name = (entity.metadata.get("name") or "").strip()
+                entity_type = entity.metadata.get("entity_type") or entity.memory_type or "concept"
+                confidence = float(entity.metadata.get("confidence", 0.0))
+
+                # Pre-write quality gates (confidence, length, blocklist)
+                if not name or len(name) < _MIN_NAME_LENGTH:
+                    continue
+                if name.lower() in _BLOCKLIST:
+                    continue
+                if confidence < _MIN_CONFIDENCE:
+                    continue
+
+                added = ontology_graph.add_entity_pattern(
+                    name=name,
+                    label=entity_type,
+                    confidence=confidence,
+                    workspace_id=workspace_id,
+                    is_global=False,
+                    source="llm_discovery",
+                )
+                if added:
+                    new_pattern_count += 1
+        except Exception as exc:
+            logger.warning("Extract job: failed to update entity patterns for item_id=%s: %s", item_id, exc)
+
+        if new_pattern_count > 0 and redis_client is not None:
+            try:
+                from smartmemory.ontology.pattern_manager import PatternManager
+
+                channel = f"{PatternManager.RELOAD_CHANNEL_PREFIX}:{workspace_id}"
+                redis_client.publish(channel, "reload")
+                logger.debug("Extract job: published ruler reload on %s", channel)
+            except Exception as exc:
+                logger.warning("Extract job: failed to publish ruler reload: %s", exc)
+    else:
+        logger.debug("Extract job: ontology disabled; skipping EntityPattern writes for item_id=%s", item_id)
+
+    logger.info(
+        "Extract job ok: item_id=%s new_entities=%d new_relations=%d new_patterns=%d",
+        item_id,
+        new_entity_count,
+        new_relation_count,
+        new_pattern_count,
+    )
+    return {
+        "status": "ok",
+        "new_entities": new_entity_count,
+        "new_relations": new_relation_count,
+        "new_patterns": new_pattern_count,
+    }

smartmemory/background/id_resolver.py

@@ -0,0 +1,72 @@
+"""Id resolution between LLMSingleExtractor's SHA-256 id space and FalkorDB stored ids.
+
+LLMSingleExtractor._process_entities() assigns SHA-256 item_ids (hashlib.sha256 of
+"name|entity_type", first 16 chars) to MemoryItem objects. StoreStage._map_entity_ids()
+assigns FalkorDB graph node ids, stored in PipelineState.entity_ids as {name -> graph_id}.
+
+Relations from the LLM call reference SHA-256 ids which must be remapped to graph ids
+before writing edges. Both id spaces are needed: sha256_to_stored for ruler entities
+present in both spaces, net_new_ids for entities added only in this Tier 2 run.
+"""
+from __future__ import annotations
+
+from smartmemory.models.memory_item import MemoryItem
+
+
+def build_sha256_to_stored(
+    llm_entities: list[MemoryItem],
+    ruler_entity_ids: dict[str, str],
+) -> dict[str, str]:
+    """Map SHA-256 ids (LLM extractor space) to stored FalkorDB ids (Tier 1 space).
+
+    Args:
+        llm_entities: List[MemoryItem] from LLMSingleExtractor.extract()["entities"].
+                      Accessed via entity.item_id (SHA-256) and entity.metadata["name"].
+        ruler_entity_ids: state.entity_ids from the job payload — {name.lower(): graph_node_id}.
+                          Populated by StoreStage._map_entity_ids() during Tier 1 ingest.
+
+    Returns:
+        {sha256_id: graph_node_id} for entities present in both id spaces.
+    """
+    remap: dict[str, str] = {}
+    for entity in llm_entities:
+        name = entity.metadata.get("name", "").lower()
+        sha256_id = entity.item_id
+        if name and sha256_id and name in ruler_entity_ids:
+            stored_id = ruler_entity_ids[name]
+            if stored_id:
+                remap[sha256_id] = stored_id
+    return remap
+
+
+def filter_valid_relations(
+    llm_relations: list[dict],
+    sha256_to_stored: dict[str, str],
+    net_new_ids: set[str],
+) -> list[dict]:
+    """Return relations where both endpoints resolve to known graph nodes.
+
+    An endpoint is valid if it appears in sha256_to_stored (ruler entity remapped
+    to a stored id) or in net_new_ids (entity written in this Tier 2 run).
+
+    Args:
+        llm_relations: List of relation dicts from LLMSingleExtractor.extract()["relations"].
+                       Each dict: {source_id: sha256, target_id: sha256, relation_type: str}.
+        sha256_to_stored: Output of build_sha256_to_stored().
+        net_new_ids: Set of item_ids for net-new entities written in this Tier 2 run.
+
+    Returns:
+        List of relation dicts with source_id/target_id remapped to graph node ids.
+    """
+    valid_ids = set(sha256_to_stored.values()) | net_new_ids
+    resolved = []
+    for rel in llm_relations:
+        source_id = sha256_to_stored.get(rel["source_id"], rel["source_id"])
+        target_id = sha256_to_stored.get(rel["target_id"], rel["target_id"])
+        if source_id in valid_ids and target_id in valid_ids:
+            resolved.append({
+                "source_id": source_id,
+                "target_id": target_id,
+                "relation_type": rel["relation_type"],
+            })
+    return resolved

smartmemory/clear_all.py

@@ -0,0 +1,130 @@
+"""
+Utility to clear SmartMemory state across caches and stores.
+
+Actions:
+- Clear SmartGraph (drops all nodes/edges, clears graph caches)
+- Clear VectorStore (if supported by current backend)
+- Clear Redis-backed extraction caches (best effort)
+
+Usage:
+  python -m smartmemory.maintenance.clear_all --all
+  python -m smartmemory.maintenance.clear_all --graph
+  python -m smartmemory.maintenance.clear_all --vector
+  python -m smartmemory.maintenance.clear_all --cache
+
+Notes:
+- This is intended for local/dev use. Use with caution in shared environments.
+- We do not swallow exceptions silently: failures are logged and surfaced.
+"""
+from __future__ import annotations
+
+import argparse
+import logging
+import os
+from typing import Any
+
+
+def _setup_logging(verbose: bool):
+    level = logging.DEBUG if verbose else logging.INFO
+    logging.basicConfig(level=level, format="[%(levelname)s] %(message)s")
+
+
+def clear_graph() -> bool:
+    from smartmemory.graph.smartgraph import SmartGraph
+
+    logging.info("Clearing SmartGraph (nodes/edges/caches)...")
+    try:
+        g = SmartGraph()
+        ok = g.clear()
+        logging.info("SmartGraph cleared: %s", ok)
+        return bool(ok)
+    except Exception as e:
+        logging.error("Failed to clear SmartGraph: %s", e)
+        raise
+
+
+def clear_vector_store() -> bool:
+    try:
+        from smartmemory.stores.vector.vector_store import VectorStore
+    except Exception as e:
+        logging.warning("VectorStore not available: %s", e)
+        return False
+
+    logging.info("Clearing VectorStore (if backend supports clear())...")
+    try:
+        vs = VectorStore()
+        ok = vs.clear()
+        logging.info("VectorStore cleared via clear(): %s", ok)
+        return bool(ok)
+    except Exception as e:
+        logging.error("Failed to clear VectorStore: %s", e)
+        raise
+
+
+def clear_cache() -> bool:
+    try:
+        from smartmemory.utils.cache import get_cache
+    except Exception as e:
+        logging.warning("Cache utils not available: %s", e)
+        return False
+
+    logging.info("Clearing Redis-backed caches (best effort)...")
+    try:
+        cache = get_cache()
+        # Try to get redis client from cache object
+        client = getattr(cache, 'redis', None)
+        if client and hasattr(client, "flushdb"):
+            client.flushdb()
+            logging.info("Redis FLUSHDB successful")
+            return True
+        # Extract connection params if available
+        url = os.getenv("REDIS_URL") or os.getenv("REDIS_URI")
+        if url:
+            try:
+                import redis  # type: ignore
+                r = redis.from_url(url)
+                r.flushdb()
+                logging.info("Redis FLUSHDB via REDIS_URL successful")
+                return True
+            except Exception as e:
+                logging.warning("Redis module/connection failed: %s", e)
+        logging.warning("Could not locate a Redis client to flush; skipped.")
+        return False
+    except Exception as e:
+        logging.error("Failed to clear caches: %s", e)
+        raise
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Clear SmartMemory state across caches and stores.")
+    parser.add_argument("--all", action="store_true", help="Clear graph, vector store, and caches")
+    parser.add_argument("--graph", action="store_true", help="Clear SmartGraph")
+    parser.add_argument("--vector", action="store_true", help="Clear VectorStore")
+    parser.add_argument("--cache", action="store_true", help="Clear Redis-backed caches")
+    parser.add_argument("-v", "--verbose", action="store_true", help="Verbose logging")
+    args = parser.parse_args()
+
+    _setup_logging(args.verbose)
+
+    if not any([args.all, args.graph, args.vector, args.cache]):
+        logging.info("No flags provided; defaulting to --all")
+        args.all = True
+
+    results: dict[str, Any] = {}
+
+    try:
+        if args.all or args.graph:
+            results["graph"] = clear_graph()
+        if args.all or args.vector:
+            results["vector"] = clear_vector_store()
+        if args.all or args.cache:
+            results["cache"] = clear_cache()
+    except Exception as e:
+        logging.error("One or more clear operations failed: %s", e)
+        raise
+
+    logging.info("Done. Summary: %s", results)
+
+
+if __name__ == "__main__":
+    main()