synapse-framework 0.2.4__py3-none-any.whl

synapse/rag/chunker.py

@@ -0,0 +1,219 @@
+"""Text chunking utilities for RAG ingestion in Synapse."""
+
+from __future__ import annotations
+
+import hashlib
+from typing import Any
+
+from langchain_core.documents import Document
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+
+from synapse.config import DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE
+
+
+class DocumentChunker:
+    """Splits arbitrary text and connector payloads into deterministic LangChain Document chunks."""
+
+    def __init__(
+        self,
+        chunk_size: int = DEFAULT_CHUNK_SIZE,
+        chunk_overlap: int = DEFAULT_CHUNK_OVERLAP,
+    ) -> None:
+        """Initializes the document chunker.
+
+        Args:
+            chunk_size: Maximum character length per chunk. Defaults to 280.
+            chunk_overlap: Character overlap between consecutive chunks. Defaults to 30.
+        """
+        self.chunk_size = chunk_size
+        self.chunk_overlap = chunk_overlap
+        self.splitter = RecursiveCharacterTextSplitter(
+            chunk_size=self.chunk_size,
+            chunk_overlap=self.chunk_overlap,
+            separators=["\n\n", "\n", ". ", "? ", "! ", " ", ""],
+            length_function=len,
+        )
+
+    @staticmethod
+    def _generate_chunk_id(source: str, parent_id: str, index: int) -> str:
+        seed = f"{source}:{parent_id}:{index}"
+        return hashlib.sha256(seed.encode("utf-8")).hexdigest()[:16]
+
+    def chunk_text(
+        self,
+        text: str,
+        source: str = "text",
+        metadata: dict[str, Any] | None = None,
+        parent_id: str | None = None,
+    ) -> list[Document]:
+        """Splits raw text into indexed LangChain Document chunks."""
+        clean_text = text.strip()
+        if not clean_text:
+            return []
+
+        base_meta = metadata.copy() if metadata else {}
+        base_meta["source"] = source
+        doc_id = (
+            parent_id
+            or hashlib.sha256(clean_text[:128].encode("utf-8")).hexdigest()[:12]
+        )
+
+        raw_chunks = self.splitter.split_text(clean_text)
+        documents: list[Document] = []
+        for idx, chunk in enumerate(raw_chunks):
+            chunk_meta = {
+                **base_meta,
+                "parent_id": doc_id,
+                "chunk_index": idx,
+                "chunk_id": self._generate_chunk_id(source, doc_id, idx),
+            }
+            documents.append(Document(page_content=chunk, metadata=chunk_meta))
+
+        return documents
+
+    def chunk_email(self, email_data: dict[str, Any]) -> list[Document]:
+        """Formats and splits an email dictionary into Document chunks."""
+        email_id = str(email_data.get("id", ""))
+        subject = str(email_data.get("subject", "No Subject")).strip()
+        sender = str(
+            email_data.get("from") or email_data.get("sender") or "Unknown Sender"
+        ).strip()
+        recipients = str(email_data.get("to", "Me")).strip()
+        date_str = str(email_data.get("date", "")).strip()
+        body = str(email_data.get("body", "")).strip()
+        is_unread = bool(email_data.get("is_unread", False))
+        read_tag = "UNREAD" if is_unread else "READ"
+
+        if not body and not subject:
+            return []
+
+        header_context = (
+            f"[EMAIL | {read_tag}] Subject: {subject} | From: {sender} | To: {recipients}"
+        )
+        full_text = f"{header_context}\nDate: {date_str}\n\n{body}"
+
+        base_meta: dict[str, Any] = {
+            "source": "gmail",
+            "email_id": email_id,
+            "subject": subject,
+            "sender": sender,
+            "recipients": recipients,
+            "date": date_str,
+            "is_unread": is_unread,
+            "type": "Email",
+        }
+
+        raw_chunks = self.splitter.split_text(full_text)
+        documents: list[Document] = []
+        for idx, chunk in enumerate(raw_chunks):
+            chunk_meta = {
+                **base_meta,
+                "chunk_index": idx,
+                "chunk_id": self._generate_chunk_id("gmail", email_id, idx),
+            }
+            documents.append(Document(page_content=chunk, metadata=chunk_meta))
+
+        return documents
+
+    def chunk_chat_turn(
+        self,
+        user_message: str,
+        assistant_response: str,
+        session_id: str,
+        turn_index: int,
+    ) -> list[Document]:
+        """Splits a single conversation turn into Document chunks."""
+        content = (
+            f"User: {user_message.strip()}\nAssistant: {assistant_response.strip()}"
+        )
+        base_meta: dict[str, Any] = {
+            "source": "chat_history",
+            "session_id": session_id,
+            "turn_index": turn_index,
+            "type": "Conversation",
+        }
+        return self.chunk_text(
+            text=content,
+            source="chat_history",
+            metadata=base_meta,
+            parent_id=f"{session_id}_{turn_index}",
+        )
+
+    def chunk_mcp_record(
+        self,
+        server_name: str,
+        tool_name: str,
+        record: dict[str, Any],
+    ) -> list[Document]:
+        """Normalizes and splits a record from any connector/MCP tool."""
+        if server_name == "gmail":
+            return self.chunk_email(record)
+
+        rec_id = str(
+            record.get("id")
+            or record.get("page_id")
+            or record.get("url")
+            or hashlib.sha256(str(record).encode("utf-8")).hexdigest()[:12]
+        )
+        title = str(
+            record.get("title")
+            or record.get("subject")
+            or record.get("name")
+            or record.get("summary")
+            or f"{server_name.upper()} Record"
+        ).strip()
+        author = str(
+            record.get("sender")
+            or record.get("from")
+            or record.get("author")
+            or record.get("owner")
+            or record.get("creator")
+            or server_name
+        ).strip()
+        date_str = str(
+            record.get("date")
+            or record.get("last_edited_time")
+            or record.get("created_at")
+            or record.get("updated_at")
+            or ""
+        ).strip()
+        body = str(
+            record.get("body")
+            or record.get("content")
+            or record.get("text")
+            or record.get("description")
+            or record.get("snippet")
+            or ""
+        ).strip()
+        url = str(record.get("url") or "").strip()
+
+        if not body and not title:
+            return []
+
+        header = f"[{server_name.upper()} | {tool_name}] Title: {title} | Source: {author}"
+        if url:
+            header += f" | URL: {url}"
+        full_text = f"{header}\nDate: {date_str}\n\n{body}".strip()
+
+        base_meta: dict[str, Any] = {
+            "source": server_name,
+            "tool": tool_name,
+            "record_id": rec_id,
+            "subject": title,
+            "sender": author,
+            "date": date_str,
+            "url": url,
+            "type": str(record.get("type") or server_name.capitalize()),
+        }
+
+        raw_chunks = self.splitter.split_text(full_text)
+        documents: list[Document] = []
+        for idx, chunk in enumerate(raw_chunks):
+            chunk_meta = {
+                **base_meta,
+                "chunk_index": idx,
+                "chunk_id": self._generate_chunk_id(server_name, rec_id, idx),
+            }
+            documents.append(Document(page_content=chunk, metadata=chunk_meta))
+
+        return documents

synapse/rag/hybrid_rag.py

@@ -0,0 +1,122 @@
+"""Hybrid vector and knowledge graph context retriever."""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+from langchain_core.documents import Document
+
+from synapse.config import DEFAULT_TOP_K
+
+
+class HybridRAGEngine:
+    """Combines dense semantic similarity search with knowledge graph multi-hop relations."""
+
+    def __init__(
+        self,
+        vector_store: Any | None = None,
+        graph_store: Any | None = None,
+    ) -> None:
+        """Initializes the Hybrid RAG engine."""
+        self.vector_store = vector_store
+        self.graph_store = graph_store
+
+    def _extract_candidate_entities(self, query: str) -> list[str]:
+        candidates = set()
+        if self.graph_store and hasattr(self.graph_store, "graph") and hasattr(self.graph_store.graph, "nodes"):
+            existing_nodes = list(self.graph_store.graph.nodes)
+            query_lower = query.lower()
+            for node in existing_nodes:
+                if str(node).lower() in query_lower:
+                    candidates.add(str(node))
+
+        matches = re.findall(r"\b[A-Z][a-zA-Z0-9_-]+\b", query)
+        for match in matches:
+            candidates.add(match)
+
+        return list(candidates)
+
+    def retrieve_context(
+        self,
+        query: str,
+        k: int = DEFAULT_TOP_K,
+        filter_dict: dict[str, Any] | None = None,
+    ) -> dict[str, Any]:
+        """Retrieves matching vector chunks and relational facts for a query."""
+        vector_docs: list[Document] = []
+        if self.vector_store is not None:
+            try:
+                vector_docs = self.vector_store.similarity_search(
+                    query=query, k=k, filter_dict=filter_dict
+                )
+            except Exception:
+                vector_docs = []
+
+        candidate_entities: list[str] = []
+        graph_facts: list[str] = []
+        if self.graph_store is not None:
+            try:
+                candidate_entities = self._extract_candidate_entities(query)
+                for entity in candidate_entities:
+                    facts = self.graph_store.find_relation_for(entity)
+                    graph_facts.extend(facts)
+            except Exception:
+                pass  # nosec B110
+
+        unique_facts = list(dict.fromkeys(graph_facts))
+        formatted_context = self._format_prompt_context(
+            vector_docs, unique_facts
+        )
+
+        serialized_chunks = [
+            {
+                "page_content": doc.page_content,
+                "metadata": doc.metadata,
+                "source": doc.metadata.get("source", "chroma_db"),
+                "subject": doc.metadata.get("subject", ""),
+                "chunk_id": doc.metadata.get("chunk_id", ""),
+            }
+            for doc in vector_docs
+        ]
+
+        return {
+            "query": query,
+            "vector_documents": vector_docs,
+            "chunks": serialized_chunks,
+            "graph_facts": unique_facts,
+            "graph_triples": unique_facts,
+            "entities_detected": candidate_entities,
+            "formatted_context": formatted_context,
+        }
+
+    @staticmethod
+    def _format_prompt_context(
+        vector_docs: list[Document],
+        graph_facts: list[str],
+    ) -> str:
+        sections: list[str] = []
+
+        if graph_facts:
+            facts_block = "\n".join(f"- {fact}" for fact in graph_facts)
+            sections.append(
+                f"### RELATIONAL KNOWLEDGE GRAPH FACTS:\n{facts_block}"
+            )
+
+        if vector_docs:
+            doc_chunks: list[str] = []
+            for idx, doc in enumerate(vector_docs, 1):
+                src = doc.metadata.get("source", "unknown")
+                subj = doc.metadata.get("subject", "")
+                header = f"[{src.upper()}" + (f" - {subj}" if subj else "") + "]"
+                doc_chunks.append(
+                    f"{idx}. {header}\n{doc.page_content.strip()}"
+                )
+            sections.append(
+                "### RELEVANT DOCUMENT EXCERPTS:\n" + "\n\n".join(doc_chunks)
+            )
+
+        if not sections:
+            return "No relevant past context or graph facts found."
+
+        return "\n\n".join(sections)

synapse/resource/__init__.py

@@ -0,0 +1,5 @@
+"""Resource module for managing document collections and RAG ingestion."""
+
+from synapse.resource.resource import Resource
+
+__all__ = ["Resource"]

synapse/resource/resource.py

@@ -0,0 +1,304 @@
+"""Resource management and document collection ingestion for Synapse."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from collections.abc import Sequence
+from pathlib import Path
+from typing import Any
+
+from langchain_core.documents import Document
+
+from synapse.rag.chunker import DocumentChunker
+
+
+class Resource:
+    """Manages collections of static or dynamic documents for Synapse RAG retrieval.
+
+    Supports ingesting raw strings, local files (.txt, .md, .pdf, .json, .csv),
+    directories, and LangChain Document instances. Automatically chunks and embeds
+    content for instant vector search and knowledge graph indexing.
+
+    Example:
+        ```python
+        from synapse.resource import Resource
+
+        # Ingest files or text
+        res = Resource("knowledge_base")
+        res.add("README.md")
+        res.add("Quick note about architecture", title="Architecture")
+        res.add(["docs/guide.md", "data/faq.txt"])
+
+        # Pass to BaseApp:
+        app = BaseApp(resource=res)
+        ```
+    """
+
+    def __init__(
+        self,
+        name: str = "resource",
+        documents: Document | str | Path | Sequence[Any] | None = None,
+        chunker: DocumentChunker | None = None,
+        metadata: dict[str, Any] | None = None,
+    ) -> None:
+        """Initializes a Resource container.
+
+        Args:
+            name: Logical identifier for this resource collection.
+            documents: Optional initial document, file path, or sequence of documents.
+            chunker: Optional custom DocumentChunker instance.
+            metadata: Arbitrary metadata associated with this resource.
+        """
+        self.name = name
+        self.chunker = chunker or DocumentChunker()
+        self.metadata = dict(metadata or {})
+        self._raw_documents: list[Document] = []
+        self._is_indexed: bool = False
+
+        if documents is not None:
+            self.add(documents)
+
+    @property
+    def documents(self) -> list[Document]:
+        """Returns the list of raw Document objects in this resource."""
+        return list(self._raw_documents)
+
+    def __len__(self) -> int:
+        return len(self._raw_documents)
+
+    def __repr__(self) -> str:
+        return f"<Resource name='{self.name}' documents={len(self._raw_documents)}>"
+
+    def add(
+        self,
+        *items: Any,
+        **extra_metadata: Any,
+    ) -> Resource:
+        """Adds one or more documents, text strings, files, or directories to this resource.
+
+        Args:
+            *items: Variadic items to add. Each item can be a LangChain Document,
+                raw text string, file Path, directory Path, or a list/tuple of them.
+            **extra_metadata: Metadata merged into all items added during this call.
+
+        Returns:
+            The current Resource instance for fluent method chaining.
+        """
+        for item in items:
+            self._ingest_item(item, extra_metadata)
+        self._is_indexed = False
+        return self
+
+    def _ingest_item(self, item: Any, extra_metadata: dict[str, Any]) -> None:
+        """Internal dispatcher for handling diverse input types."""
+        if item is None:
+            return
+
+        if isinstance(item, (list, tuple, set)):
+            for sub_item in item:
+                self._ingest_item(sub_item, extra_metadata)
+            return
+
+        if isinstance(item, Document):
+            merged_meta = {**item.metadata, **extra_metadata, "resource": self.name}
+            self._raw_documents.append(
+                Document(page_content=item.page_content, metadata=merged_meta)
+            )
+            return
+
+        if isinstance(item, Path):
+            self._ingest_path(item, extra_metadata)
+            return
+
+        if isinstance(item, str):
+            clean_str = item.strip()
+            # Check if this string points to an existing file or directory
+            candidate = Path(clean_str)
+            if candidate.exists():
+                self._ingest_path(candidate, extra_metadata)
+                return
+
+            # Treat as raw text
+            doc_id = hashlib.sha256(clean_str[:256].encode("utf-8")).hexdigest()[:12]
+            meta = {
+                "source": "resource_text",
+                "resource": self.name,
+                "record_id": doc_id,
+                **extra_metadata,
+            }
+            self._raw_documents.append(
+                Document(page_content=clean_str, metadata=meta)
+            )
+            return
+
+        if isinstance(item, dict):
+            content = str(
+                item.get("content")
+                or item.get("text")
+                or item.get("body")
+                or item.get("page_content")
+                or json.dumps(item)
+            )
+            meta = {
+                "source": "resource_dict",
+                "resource": self.name,
+                **item.get("metadata", {}),
+                **extra_metadata,
+            }
+            self._raw_documents.append(Document(page_content=content, metadata=meta))
+            return
+
+        # Fallback for any other object: convert to string
+        self._raw_documents.append(
+            Document(
+                page_content=str(item),
+                metadata={"source": "resource_object", "resource": self.name, **extra_metadata},
+            )
+        )
+
+    def _ingest_path(self, path: Path, extra_metadata: dict[str, Any]) -> None:
+        """Reads a file or directory from disk."""
+        resolved = path.resolve()
+        if not resolved.exists():
+            raise FileNotFoundError(f"Resource path does not exist: {resolved}")
+
+        if resolved.is_dir():
+            # Ingest all supported files in directory
+            extensions = {".md", ".txt", ".json", ".csv", ".py", ".html", ".log", ".yaml", ".yml"}
+            for child in sorted(resolved.rglob("*")):
+                if child.is_file() and child.suffix.lower() in extensions:
+                    self._ingest_file(child, extra_metadata)
+            return
+
+        if resolved.is_file():
+            self._ingest_file(resolved, extra_metadata)
+
+    def _ingest_file(self, file_path: Path, extra_metadata: dict[str, Any]) -> None:
+        """Reads a single file into a Document."""
+        suffix = file_path.suffix.lower()
+        title = file_path.stem.replace("_", " ").replace("-", " ").title()
+
+        try:
+            content = file_path.read_text(encoding="utf-8", errors="ignore")
+        except Exception as e:
+            content = f"Error reading file {file_path.name}: {e}"
+
+        meta = {
+            "source": str(file_path),
+            "filename": file_path.name,
+            "title": title,
+            "subject": title,
+            "resource": self.name,
+            "file_type": suffix.lstrip("."),
+            **extra_metadata,
+        }
+
+        self._raw_documents.append(Document(page_content=content, metadata=meta))
+
+    def add_text(
+        self,
+        text: str,
+        title: str | None = None,
+        source: str = "text",
+        **metadata: Any,
+    ) -> Resource:
+        """Convenience helper to add raw text with a title."""
+        meta = {"source": source, **metadata}
+        if title:
+            meta["title"] = title
+            meta["subject"] = title
+        return self.add(text, **meta)
+
+    def add_file(
+        self,
+        file_path: str | Path,
+        **metadata: Any,
+    ) -> Resource:
+        """Convenience helper to add a single file."""
+        return self.add(Path(file_path), **metadata)
+
+    def add_directory(
+        self,
+        dir_path: str | Path,
+        **metadata: Any,
+    ) -> Resource:
+        """Convenience helper to ingest an entire directory."""
+        return self.add(Path(dir_path), **metadata)
+
+    def get_chunks(self) -> list[Document]:
+        """Splits all ingested raw documents into indexable chunks using DocumentChunker."""
+        all_chunks: list[Document] = []
+        for doc in self._raw_documents:
+            src = doc.metadata.get("source", self.name)
+            rec_id = doc.metadata.get("record_id") or hashlib.sha256(
+                doc.page_content[:128].encode("utf-8")
+            ).hexdigest()[:12]
+            chunks = self.chunker.chunk_text(
+                text=doc.page_content,
+                source=src,
+                metadata=doc.metadata,
+                parent_id=rec_id,
+            )
+            all_chunks.extend(chunks)
+        return all_chunks
+
+    def index_into(
+        self,
+        vector_store: Any | None = None,
+        graph_store: Any | None = None,
+    ) -> int:
+        """Chunks and indexes this resource into vector store and knowledge graph.
+
+        Args:
+            vector_store: Optional vector store instance (LocalVectorStore, VectorStorage, BaseVectorDB).
+            graph_store: Optional graph store instance (PersistentGraphStore, BaseGraphStore).
+
+        Returns:
+            The total number of chunks successfully processed.
+        """
+        chunks = self.get_chunks()
+        if not chunks:
+            return 0
+
+        if vector_store is not None:
+            vector_store.add_documents(chunks)
+
+        if graph_store is not None:
+            # Register resource node and links in knowledge graph
+            graph_store.add_node_entity(
+                name=self.name,
+                entity_type="Resource",
+                attributes={"total_docs": len(self._raw_documents), "chunks": len(chunks)},
+            )
+            for doc in self._raw_documents:
+                title = (
+                    doc.metadata.get("title")
+                    or doc.metadata.get("filename")
+                    or doc.metadata.get("subject")
+                    or (doc.page_content[:35].strip() + "...")
+                )
+                if title:
+                    graph_store.add_relation(
+                        source=self.name,
+                        relation="CONTAINS",
+                        target=str(title),
+                        source_type="Resource",
+                        target_type="Document",
+                    )
+
+        self._is_indexed = True
+        return len(chunks)
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serializes resource statistics into a dictionary."""
+        return {
+            "name": self.name,
+            "document_count": len(self._raw_documents),
+            "chunk_count": len(self.get_chunks()),
+            "is_indexed": self._is_indexed,
+            "metadata": self.metadata,
+        }
+
+
+__all__ = ["Resource"]

synapse/resources/__init__.py

@@ -0,0 +1,5 @@
+"""Alias package for synapse.resource."""
+
+from synapse.resource.resource import Resource
+
+__all__ = ["Resource"]

synapse/skills/__init__.py

@@ -0,0 +1,5 @@
+"""Skills module for specialized agent instructions and domain personas."""
+
+from synapse.skills.skill import Skill, make_skill, makeskills
+
+__all__ = ["Skill", "make_skill", "makeskills"]