memory-access 0.1.3__py3-none-any.whl

memory_access/__init__.py

@@ -0,0 +1 @@
+"""Semantic Memory MCP Server — intent-based memory storage and retrieval for AI agents."""

memory_access/cli.py

@@ -0,0 +1,131 @@
+from __future__ import annotations
+
+import argparse
+import asyncio
+import sys
+
+
+def main():
+    """Entry point for memory-access CLI."""
+    if len(sys.argv) > 1 and sys.argv[1] == "kb":
+        return _run_kb_cli()
+
+    # Default: run MCP server
+    from .server import main as server_main
+    server_main()
+
+
+def _run_kb_cli():
+    parser = argparse.ArgumentParser(prog="memory-access kb")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    # new
+    new_p = sub.add_parser("new", help="Create a new knowledge base")
+    new_p.add_argument("name", help="Knowledge base name (slug)")
+    new_p.add_argument("--crawl", help="URL to crawl")
+    new_p.add_argument("--scrape", help="Single URL to scrape")
+    new_p.add_argument("--from-dir", dest="from_dir", help="Directory of Firecrawl JSON files")
+    new_p.add_argument("--limit", type=int, default=1000, help="Max pages to crawl")
+    new_p.add_argument("--description", default="", help="KB description")
+
+    # list
+    sub.add_parser("list", help="List knowledge bases")
+
+    # delete
+    del_p = sub.add_parser("delete", help="Delete a knowledge base")
+    del_p.add_argument("name", help="Knowledge base name")
+
+    # refresh
+    ref_p = sub.add_parser("refresh", help="Re-crawl and refresh a knowledge base")
+    ref_p.add_argument("name", help="Knowledge base name")
+    ref_p.add_argument("--limit", type=int, default=1000, help="Max pages to crawl")
+
+    args = parser.parse_args(sys.argv[2:])
+    asyncio.run(_dispatch(args))
+
+
+async def _dispatch(args):
+    from .server import create_app
+    from .ingest import Ingestor
+
+    app = await create_app()
+
+    # Only create crawl service when needed (requires firecrawl dependency + API key)
+    crawl_service = None
+    needs_crawl = args.command == "new" and (getattr(args, "crawl", None) or getattr(args, "scrape", None))
+    needs_crawl = needs_crawl or args.command == "refresh"
+    if needs_crawl:
+        from .crawl import create_crawl_service
+        crawl_service = create_crawl_service()
+
+    ingestor = Ingestor(
+        store=app.store,
+        normalizer=app.normalizer,
+        embeddings=app.embeddings,
+        crawl_service=crawl_service,
+    )
+
+    if args.command == "new":
+        await _cmd_new(app, ingestor, args)
+    elif args.command == "list":
+        await _cmd_list(app)
+    elif args.command == "delete":
+        await _cmd_delete(app, args)
+    elif args.command == "refresh":
+        await _cmd_refresh(app, ingestor, args)
+
+
+async def _cmd_new(app, ingestor, args):
+    source_type = "crawl" if args.crawl else "scrape" if args.scrape else "file" if args.from_dir else "text"
+    kb_id = await app.store.create_kb(args.name, description=args.description, source_type=source_type)
+    print(f"Created knowledge base '{args.name}' ({kb_id})", file=sys.stderr)
+
+    def on_progress(current, total, url):
+        print(f"  [{current}/{total}] {url}", file=sys.stderr)
+
+    if args.crawl:
+        count = await ingestor.ingest_crawl(kb_id, args.crawl, limit=args.limit, on_progress=on_progress)
+        print(f"Ingested {count} chunks from {args.crawl}", file=sys.stderr)
+    elif args.scrape:
+        count = await ingestor.ingest_scrape(kb_id, args.scrape)
+        print(f"Ingested {count} chunks from {args.scrape}", file=sys.stderr)
+    elif args.from_dir:
+        count = await ingestor.ingest_from_directory(kb_id, args.from_dir, on_progress=on_progress)
+        print(f"Ingested {count} chunks from {args.from_dir}", file=sys.stderr)
+
+
+async def _cmd_list(app):
+    kbs = await app.store.list_kbs()
+    if not kbs:
+        print("No knowledge bases found.", file=sys.stderr)
+        return
+    for kb in kbs:
+        desc = f" - {kb.description}" if kb.description else ""
+        print(f"  {kb.name}{desc} [{kb.source_type or 'unknown'}]", file=sys.stderr)
+
+
+async def _cmd_delete(app, args):
+    kb = await app.store.get_kb_by_name(args.name)
+    if kb is None:
+        print(f"Knowledge base '{args.name}' not found.", file=sys.stderr)
+        return
+    await app.store.delete_kb(kb.id)
+    print(f"Deleted knowledge base '{args.name}'.", file=sys.stderr)
+
+
+async def _cmd_refresh(app, ingestor, args):
+    kb = await app.store.get_kb_by_name(args.name)
+    if kb is None:
+        print(f"Knowledge base '{args.name}' not found.", file=sys.stderr)
+        return
+    if not kb.source_type or kb.source_type == "text":
+        print(f"Cannot refresh KB '{args.name}': no crawl/scrape source.", file=sys.stderr)
+        return
+
+    deleted = await app.store.delete_kb_chunks(kb.id)
+    print(f"Deleted {deleted} existing chunks.", file=sys.stderr)
+
+    # Re-ingest — we need the original URL, but it's not stored on the KB model.
+    # For now, require --crawl or --scrape on refresh too, or look for source_url from old chunks.
+    # Since we don't store the source URL on the KB, print a message.
+    print(f"Chunks cleared. Use 'memory-access kb new' with --crawl/--scrape to re-ingest.", file=sys.stderr)

memory_access/crawl.py

@@ -0,0 +1,84 @@
+from __future__ import annotations
+
+import os
+from abc import ABC, abstractmethod
+
+from .models import CrawledPage
+
+
+class CrawlService(ABC):
+    """Abstract crawl service. Implement for each crawl provider."""
+
+    @abstractmethod
+    async def crawl(self, url: str, limit: int = 1000) -> list[CrawledPage]:
+        """Crawl a URL and return pages as markdown."""
+        ...
+
+    @abstractmethod
+    async def scrape(self, url: str) -> CrawledPage:
+        """Scrape a single URL and return as markdown."""
+        ...
+
+
+class FirecrawlService(CrawlService):
+    """Crawl service using Firecrawl API."""
+
+    def __init__(self, api_key: str | None = None):
+        from firecrawl import FirecrawlApp
+
+        self.app = FirecrawlApp(api_key=api_key or os.environ.get("FIRECRAWL_API_KEY"))
+
+    async def crawl(self, url: str, limit: int = 1000) -> list[CrawledPage]:
+        """Crawl a URL using Firecrawl. Returns markdown pages."""
+        result = self.app.crawl(
+            url,
+            limit=limit,
+            scrape_options={"formats": ["markdown"], "only_main_content": True},
+        )
+
+        pages = []
+        for doc in result.data:
+            # Extract URL from metadata, fallback to base URL
+            page_url = url
+            if doc.metadata and doc.metadata.url:
+                page_url = doc.metadata.url
+            elif doc.metadata and doc.metadata.source_url:
+                page_url = doc.metadata.source_url
+
+            pages.append(
+                CrawledPage(
+                    url=page_url,
+                    markdown=doc.markdown or "",
+                    metadata=doc.metadata.model_dump(exclude_none=True) if doc.metadata else {},
+                )
+            )
+        return pages
+
+    async def scrape(self, url: str) -> CrawledPage:
+        """Scrape a single URL using Firecrawl."""
+        result = self.app.scrape(
+            url,
+            formats=["markdown"],
+            only_main_content=True,
+        )
+
+        # Extract URL from metadata, fallback to input URL
+        page_url = url
+        if result.metadata and result.metadata.url:
+            page_url = result.metadata.url
+        elif result.metadata and result.metadata.source_url:
+            page_url = result.metadata.source_url
+
+        return CrawledPage(
+            url=page_url,
+            markdown=result.markdown or "",
+            metadata=result.metadata.model_dump(exclude_none=True) if result.metadata else {},
+        )
+
+
+def create_crawl_service(provider: str | None = None, **kwargs) -> CrawlService:
+    """Factory to create the appropriate crawl service."""
+    provider = provider or os.environ.get("CRAWL_SERVICE", "firecrawl")
+    if provider == "firecrawl":
+        return FirecrawlService(**kwargs)
+    raise ValueError(f"Unknown crawl service: {provider}")

memory_access/embeddings.py

@@ -0,0 +1,96 @@
+import json
+import os
+
+import numpy as np
+import openai
+
+
+class EmbeddingEngine:
+    """Generates normalized embeddings using OpenAI's text-embedding-3-small model."""
+
+    def __init__(self, model: str = "text-embedding-3-small"):
+        self._client: openai.OpenAI | None = None
+        self._model = model
+
+    @property
+    def client(self) -> openai.OpenAI:
+        if self._client is None:
+            self._client = openai.OpenAI(api_key=os.environ.get("OPENAI_API_KEY"))
+        return self._client
+
+    def embed(self, text: str) -> np.ndarray:
+        response = self.client.embeddings.create(input=[text], model=self._model)
+        vec = np.array(response.data[0].embedding, dtype=np.float32)
+        norm = np.linalg.norm(vec)
+        return vec / norm if norm > 0 else vec
+
+    def embed_batch(self, texts: list[str]) -> np.ndarray:
+        response = self.client.embeddings.create(input=texts, model=self._model)
+        vecs = np.array([d.embedding for d in response.data], dtype=np.float32)
+        norms = np.linalg.norm(vecs, axis=1, keepdims=True)
+        norms[norms == 0] = 1
+        return vecs / norms
+
+
+class BedrockEmbeddingEngine:
+    """Generates normalized embeddings using Amazon Titan via AWS Bedrock."""
+
+    def __init__(
+        self,
+        model: str | None = None,
+        aws_region: str | None = None,
+        aws_profile: str | None = None,
+    ):
+        self._client = None
+        self._model = model or os.environ.get(
+            "BEDROCK_EMBEDDING_MODEL", "amazon.titan-embed-text-v2:0"
+        )
+        self._aws_region = aws_region or os.environ.get("AWS_REGION", "us-east-1")
+        self._aws_profile = aws_profile or os.environ.get("AWS_PROFILE")
+
+    @property
+    def client(self):
+        if self._client is None:
+            import boto3
+
+            session_kwargs = {"region_name": self._aws_region}
+            if self._aws_profile:
+                session_kwargs["profile_name"] = self._aws_profile
+            session = boto3.Session(**session_kwargs)
+            self._client = session.client("bedrock-runtime")
+        return self._client
+
+    def _invoke(self, text: str) -> list[float]:
+        body = json.dumps({"inputText": text})
+        response = self.client.invoke_model(
+            modelId=self._model,
+            contentType="application/json",
+            accept="application/json",
+            body=body,
+        )
+        result = json.loads(response["body"].read())
+        return result["embedding"]
+
+    def embed(self, text: str) -> np.ndarray:
+        vec = np.array(self._invoke(text), dtype=np.float32)
+        norm = np.linalg.norm(vec)
+        return vec / norm if norm > 0 else vec
+
+    def embed_batch(self, texts: list[str]) -> np.ndarray:
+        import concurrent.futures
+        with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
+            results = list(executor.map(self._invoke, texts))
+        vecs = np.array(results, dtype=np.float32)
+        norms = np.linalg.norm(vecs, axis=1, keepdims=True)
+        norms[norms == 0] = 1
+        return vecs / norms
+
+
+def create_embedding_engine(
+    provider: str | None = None, **kwargs
+) -> EmbeddingEngine | BedrockEmbeddingEngine:
+    """Factory to create the appropriate embedding engine based on provider."""
+    provider = provider or os.environ.get("EMBEDDING_PROVIDER", "openai")
+    if provider == "bedrock":
+        return BedrockEmbeddingEngine(**kwargs)
+    return EmbeddingEngine(**kwargs)

memory_access/ingest.py

@@ -0,0 +1,228 @@
+from __future__ import annotations
+
+import json
+import os
+import logging
+from pathlib import Path
+
+from .crawl import CrawlService
+from .embeddings import EmbeddingEngine, BedrockEmbeddingEngine
+from .models import CrawledPage, KbChunk
+from .normalizer import Normalizer
+from .storage import InsightStore
+
+logger = logging.getLogger(__name__)
+
+
+def clean_markdown(text: str) -> str:
+    """Strip common boilerplate from crawled markdown.
+
+    Removes navigation headers (before first # heading) and
+    feedback footers ("Did you find this page useful?" etc).
+    """
+    lines = text.split("\n")
+
+    # Find first H1 heading — content starts there
+    start = 0
+    for i, line in enumerate(lines):
+        if line.startswith("# "):
+            start = i
+            break
+
+    # Find footer markers — content ends before them
+    end = len(lines)
+    footer_markers = [
+        "Did you find this page useful",
+        "Thanks for rating this page",
+        "Report a problem on this page",
+    ]
+    for i, line in enumerate(lines[start:], start):
+        if any(marker in line for marker in footer_markers):
+            end = i
+            break
+
+    return "\n".join(lines[start:end]).strip()
+
+
+def split_markdown(text: str, max_chars: int = 4000) -> list[str]:
+    """Split markdown into chunks by ## headings, with max_chars fallback.
+
+    Strategy:
+    1. Split on ## headings — each section becomes a chunk
+    2. If a section exceeds max_chars, split on paragraphs (double newline)
+    3. If a paragraph still exceeds max_chars, split at max_chars boundary
+    """
+    if not text.strip():
+        return []
+
+    # Split on ## headings, preserving the heading with its content
+    sections = []
+    current = []
+    for line in text.split("\n"):
+        if line.startswith("## ") and current:
+            sections.append("\n".join(current))
+            current = [line]
+        else:
+            current.append(line)
+    if current:
+        sections.append("\n".join(current))
+
+    # Sub-split oversized sections
+    chunks = []
+    for section in sections:
+        if len(section) <= max_chars:
+            chunks.append(section)
+        else:
+            # Split on paragraphs
+            paragraphs = section.split("\n\n")
+            current_chunk = ""
+            for para in paragraphs:
+                if len(current_chunk) + len(para) + 2 > max_chars:
+                    if current_chunk:
+                        chunks.append(current_chunk)
+                    # Handle single paragraphs exceeding max_chars
+                    if len(para) > max_chars:
+                        for i in range(0, len(para), max_chars):
+                            chunks.append(para[i:i + max_chars])
+                        current_chunk = ""
+                    else:
+                        current_chunk = para
+                else:
+                    current_chunk = current_chunk + "\n\n" + para if current_chunk else para
+            if current_chunk:
+                chunks.append(current_chunk)
+
+    return [c.strip() for c in chunks if c.strip()]
+
+
+class Ingestor:
+    """Orchestrates: crawl -> split -> normalize -> embed -> store."""
+
+    def __init__(
+        self,
+        store: InsightStore,
+        normalizer: Normalizer,
+        embeddings: EmbeddingEngine | BedrockEmbeddingEngine,
+        crawl_service: CrawlService | None = None,
+    ):
+        self.store = store
+        self.normalizer = normalizer
+        self.embeddings = embeddings
+        self.crawl_service = crawl_service
+
+    async def ingest_crawl(
+        self,
+        kb_id: str,
+        url: str,
+        limit: int = 1000,
+        on_progress: callable | None = None,
+    ) -> int:
+        """Crawl a URL and ingest all pages into a knowledge base.
+
+        Returns the total number of chunks stored.
+        """
+        pages = await self.crawl_service.crawl(url, limit=limit)
+        total_chunks = 0
+
+        for i, page in enumerate(pages):
+            if on_progress:
+                on_progress(i + 1, len(pages), page.url)
+
+            chunks_stored = await self.ingest_page(kb_id, page)
+            total_chunks += chunks_stored
+
+        return total_chunks
+
+    async def ingest_page(self, kb_id: str, page: CrawledPage) -> int:
+        """Ingest a single crawled page into a knowledge base.
+
+        Returns the number of chunks stored.
+        """
+        cleaned = clean_markdown(page.markdown)
+        text_chunks = split_markdown(cleaned)
+
+        # Collect all insights from all chunks
+        all_insights = []
+        for chunk_text in text_chunks:
+            try:
+                insights = await self.normalizer.normalize(chunk_text)
+                all_insights.extend(insights)
+            except Exception as e:
+                logger.warning("Failed to normalize chunk from %s: %s", page.url, e)
+                continue
+
+        if not all_insights:
+            return 0
+
+        # Filter low-confidence insights
+        min_threshold = float(os.environ.get("MIN_CONFIDENCE_THRESHOLD", "0.5"))
+        filtered = [i for i in all_insights if i.confidence >= min_threshold]
+        if len(all_insights) != len(filtered):
+            logger.info(
+                "Filtered %d/%d insights below confidence threshold %.2f",
+                len(all_insights) - len(filtered), len(all_insights), min_threshold,
+            )
+        all_insights = filtered
+
+        if not all_insights:
+            return 0
+
+        # Batch embed all normalized texts in single API call
+        texts_to_embed = [i.normalized_text for i in all_insights]
+        embeddings = self.embeddings.embed_batch(texts_to_embed)
+
+        # Store with corresponding embeddings
+        stored = 0
+        for insight, emb in zip(all_insights, embeddings):
+            kb_chunk = KbChunk(
+                kb_id=kb_id,
+                text=insight.text,
+                normalized_text=insight.normalized_text,
+                frame=insight.frame,
+                domains=insight.domains,
+                entities=insight.entities,
+                problems=insight.problems,
+                resolutions=insight.resolutions,
+                contexts=insight.contexts,
+                confidence=insight.confidence,
+                source_url=page.url,
+            )
+            await self.store.insert_kb_chunk(kb_chunk, emb)
+            stored += 1
+
+        return stored
+
+    async def ingest_scrape(self, kb_id: str, url: str) -> int:
+        """Scrape a single URL and ingest into a knowledge base."""
+        page = await self.crawl_service.scrape(url)
+        return await self.ingest_page(kb_id, page)
+
+    async def ingest_from_directory(
+        self,
+        kb_id: str,
+        dir_path: str,
+        on_progress: callable | None = None,
+    ) -> int:
+        """Load Firecrawl JSON files from a directory and ingest into a KB.
+
+        Each JSON file should have {"markdown": "...", "metadata": {"sourceURL": "..."}}.
+        Returns total chunks stored.
+        """
+        path = Path(dir_path)
+        files = sorted(path.glob("*.json"))
+        total_chunks = 0
+
+        for i, f in enumerate(files):
+            data = json.loads(f.read_text())
+            markdown = data.get("markdown", "")
+            metadata = data.get("metadata", {})
+            url = metadata.get("sourceURL") or metadata.get("url", f.stem)
+
+            if on_progress:
+                on_progress(i + 1, len(files), url)
+
+            page = CrawledPage(url=url, markdown=markdown, metadata=metadata)
+            chunks_stored = await self.ingest_page(kb_id, page)
+            total_chunks += chunks_stored
+
+        return total_chunks

memory_access/models.py

@@ -0,0 +1,86 @@
+from enum import Enum
+from datetime import datetime
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+
+class Frame(str, Enum):
+    """Canonical semantic frames for normalizing insights."""
+
+    CAUSAL = "causal"
+    CONSTRAINT = "constraint"
+    PATTERN = "pattern"
+    EQUIVALENCE = "equivalence"
+    TAXONOMY = "taxonomy"
+    PROCEDURE = "procedure"
+
+
+class Insight(BaseModel):
+    """A single atomic insight stored in semantic memory."""
+
+    id: Optional[str] = None
+    text: str
+    normalized_text: str = ""
+    frame: Frame = Frame.CAUSAL
+    domains: list[str] = Field(default_factory=list)
+    entities: list[str] = Field(default_factory=list)
+    problems: list[str] = Field(default_factory=list)
+    resolutions: list[str] = Field(default_factory=list)
+    contexts: list[str] = Field(default_factory=list)
+    confidence: float = 1.0
+    source: str = ""
+    created_at: Optional[datetime] = None
+    updated_at: Optional[datetime] = None
+
+
+class GitContext(BaseModel):
+    """Optional git metadata to associate with insights."""
+
+    repo: str = ""
+    pr: str = ""
+    author: str = ""
+    project: str = ""
+    task: str = ""
+
+
+class SearchResult(BaseModel):
+    """An insight with its similarity score from a search query."""
+
+    insight: Insight
+    score: float
+
+
+class KnowledgeBase(BaseModel):
+    """A collection of document chunks from an external source."""
+    id: Optional[str] = None
+    name: str
+    description: str = ""
+    source_type: str = ""  # 'crawl', 'scrape', 'file', 'text'
+    created_at: Optional[datetime] = None
+    updated_at: Optional[datetime] = None
+
+
+class KbChunk(BaseModel):
+    """A normalized chunk from a knowledge base document."""
+    id: Optional[str] = None
+    kb_id: str
+    text: str
+    normalized_text: str = ""
+    frame: Frame = Frame.CAUSAL
+    domains: list[str] = Field(default_factory=list)
+    entities: list[str] = Field(default_factory=list)
+    problems: list[str] = Field(default_factory=list)
+    resolutions: list[str] = Field(default_factory=list)
+    contexts: list[str] = Field(default_factory=list)
+    confidence: float = 1.0
+    source_url: str = ""
+    created_at: Optional[datetime] = None
+    updated_at: Optional[datetime] = None
+
+
+class CrawledPage(BaseModel):
+    """A single page returned by a crawl service."""
+    url: str
+    markdown: str
+    metadata: dict = Field(default_factory=dict)