cli-memory-os 0.1.0__py3-none-any.whl

connectors/gmail.py

@@ -0,0 +1,116 @@
+from composio import Composio
+from models.memory import Email
+from storage.db import insert_email, get_connection
+
+import os
+import sys
+
+def sync_gmail():
+    try:
+        try:
+            sys.stdout.reconfigure(encoding='utf-8')
+        except Exception:
+            pass
+            
+        c = Composio()
+        user_id = os.getenv("COMPOSIO_USER_ID", "user_123")
+        s = c.create(user_id=user_id)
+        
+        # Verify connection
+        toolkits_info = s.toolkits()
+        gmail_tk = next((t for t in toolkits_info.items if t.slug == "gmail"), None)
+        if not gmail_tk or not (gmail_tk.connection and gmail_tk.connection.is_active):
+            print("Gmail connection not active.")
+            return
+
+        # Fetch emails
+        resp = s.execute(tool_slug="gmail_fetch_emails", arguments={"max_results": 50, "include_payload": True})
+        if not resp or resp.error or not resp.data:
+            print("No emails found.")
+            return
+
+        emails = resp.data.get("messages", [])
+        if not emails:
+            # Check response_data structure just in case
+            emails = resp.data.get("response_data", {}).get("messages", [])
+            if not emails:
+                print("No emails found.")
+                return
+
+        print(f"Found {len(emails)} emails")
+        
+        conn = get_connection()
+        cursor = conn.cursor()
+        
+        synced_count = 0
+        skipped_count = 0
+
+        for email in emails:
+            message_id = email.get("id") or email.get("messageId") or email.get("threadId") or ""
+            
+            # Check if email is already in SQLite
+            if message_id:
+                cursor.execute("SELECT id FROM emails WHERE message_id = ?", (message_id,))
+                if cursor.fetchone():
+                    skipped_count += 1
+                    continue
+            
+            subject = email.get("subject") or "No Subject"
+            sender = email.get("sender") or email.get("from") or "Unknown Sender"
+            snippet = email.get("messageText") or email.get("snippet") or ""
+            received_at = email.get("messageTimestamp") or ""
+            date_str = received_at[:10] if len(received_at) >= 10 else received_at
+
+            # Save to SQLite
+            db_email = Email(
+                message_id=message_id,
+                subject=subject,
+                sender=sender,
+                snippet=snippet,
+                received_at=received_at
+            )
+            insert_email(db_email)
+            synced_count += 1
+
+            # Print to terminal
+            print("--------------------------------------------------")
+            print(f"Subject: {subject}")
+            print(f"From: {sender}")
+            print(f"Date: {date_str}")
+            print("--------------------------------------------------")
+        
+        conn.close()
+        print(f"Gmail Sync Complete: {synced_count} emails synced, {skipped_count} emails skipped.")
+            
+    except Exception as e:
+        print(f"Error during Gmail sync: {e}")
+
+
+from connectors.base import BaseConnector
+from connectors.registry import register
+
+@register
+class GmailConnector(BaseConnector):
+    name = "Gmail"
+    slug = "gmail"
+
+    def authenticate(self) -> bool:
+        try:
+            c = Composio()
+            user_id = os.getenv("COMPOSIO_USER_ID", "user_123")
+            s = c.create(user_id=user_id)
+            toolkits_info = s.toolkits()
+            tk = next((t for t in toolkits_info.items if t.slug == "gmail"), None)
+            return bool(tk and tk.connection and tk.connection.is_active)
+        except Exception:
+            return False
+
+    def sync(self) -> dict:
+        sync_gmail()
+        return {"status": "success"}
+
+    def health(self) -> tuple[bool, str]:
+        if self.authenticate():
+            return True, "Connected"
+        return False, "Not connected"
+

connectors/notion.py

@@ -0,0 +1,156 @@
+import sys
+from datetime import datetime
+from composio import Composio
+from models.memory import RepositoryDocument
+from storage.db import insert_repository_document, get_connection
+
+def get_page_title(page: dict) -> str:
+    properties = page.get("properties", {}) or {}
+    # Title is usually in 'title' field or 'Name' field
+    title_text = ""
+    for field_name in ["title", "Name", "name"]:
+        prop = properties.get(field_name)
+        if isinstance(prop, dict):
+            title_list = prop.get("title") or prop.get("rich_text") or []
+            if isinstance(title_list, list) and title_list:
+                title_text = "".join([t.get("plain_text", "") for t in title_list if isinstance(t, dict)])
+                if title_text:
+                    break
+    if not title_text:
+        title_text = f"Notion Page {page.get('id')}"
+    return title_text
+
+import os
+
+def sync_notion():
+    try:
+        try:
+            sys.stdout.reconfigure(encoding='utf-8')
+        except Exception:
+            pass
+
+        print("Syncing Notion...\n")
+        c = Composio()
+        user_id = os.getenv("COMPOSIO_USER_ID", "user_123")
+        s = c.create(user_id=user_id)
+
+        # Verify Notion toolkit is active
+        toolkits_info = s.toolkits()
+        notion_tk = next((t for t in toolkits_info.items if t.slug == "notion"), None)
+        if not notion_tk or not (notion_tk.connection and notion_tk.connection.is_active):
+            print("Notion connection not active.")
+            return
+
+        # Fetch page list via Composio
+        resp = s.execute(tool_slug="notion_search_notion_page", arguments={"query": ""})
+        if resp.error or not resp.data:
+            print("No Notion pages found.")
+            return
+
+        resp_data = resp.data
+        if "response_data" in resp_data:
+            resp_data = resp_data["response_data"]
+
+        pages = resp_data.get("results") or []
+        if not pages:
+            print("No Notion pages found in results.")
+            return
+
+        print(f"Found {len(pages)} Notion pages")
+
+        conn = get_connection()
+        cursor = conn.cursor()
+
+        synced_count = 0
+        skipped_count = 0
+
+        for page in pages:
+            if not isinstance(page, dict):
+                continue
+            
+            page_id = page.get("id")
+            if not page_id:
+                continue
+
+            title = get_page_title(page)
+            last_edited_time = page.get("last_edited_time") or ""
+
+            # Incremental check: check if already in DB with same last_edited_time
+            cursor.execute(
+                "SELECT content, synced_at FROM repository_documents WHERE repo_name = 'Notion' AND file_name = ?",
+                (title,)
+            )
+            row = cursor.fetchone()
+            if row:
+                stored_synced_at = row[1]
+                # If we have stored it and it wasn't edited since our last sync, skip it
+                if stored_synced_at and last_edited_time and stored_synced_at >= last_edited_time:
+                    skipped_count += 1
+                    continue
+
+            # Fetch page content
+            try:
+                print(f"Syncing page: {title}")
+                resp_content = s.execute(
+                    tool_slug="notion_get_page_markdown",
+                    arguments={"page_id": page_id}
+                )
+                if resp_content and not resp_content.error and resp_content.data:
+                    content_data = resp_content.data
+                    if "response_data" in content_data:
+                        content_data = content_data["response_data"]
+                    
+                    markdown = content_data.get("markdown") or ""
+                    if markdown.strip():
+                        # Save to database
+                        doc = RepositoryDocument(
+                            repo_name="Notion",
+                            file_name=title,
+                            content=markdown,
+                            source="notion_get_page_markdown",
+                            synced_at=datetime.now().isoformat()
+                        )
+                        insert_repository_document(doc)
+                        synced_count += 1
+                    else:
+                        print(f"Page '{title}' content was empty.")
+                else:
+                    print(f"Failed to fetch content for page: {title}")
+            except Exception as e:
+                print(f"Error fetching page content for '{title}': {e}")
+
+        conn.close()
+        print(f"Notion Sync Complete: {synced_count} pages synced, {skipped_count} pages skipped.")
+
+    except Exception as e:
+        print(f"Error during Notion sync: {e}")
+
+
+from connectors.base import BaseConnector
+from connectors.registry import register
+
+@register
+class NotionConnector(BaseConnector):
+    name = "Notion"
+    slug = "notion"
+
+    def authenticate(self) -> bool:
+        try:
+            c = Composio()
+            user_id = os.getenv("COMPOSIO_USER_ID", "user_123")
+            s = c.create(user_id=user_id)
+            toolkits_info = s.toolkits()
+            tk = next((t for t in toolkits_info.items if t.slug == "notion"), None)
+            return bool(tk and tk.connection and tk.connection.is_active)
+        except Exception:
+            return False
+
+    def sync(self) -> dict:
+        sync_notion()
+        return {"status": "success"}
+
+    def health(self) -> tuple[bool, str]:
+        if self.authenticate():
+            return True, "Connected"
+        return False, "Not connected"
+

connectors/registry.py

@@ -0,0 +1,60 @@
+"""
+Connector registry.
+
+Discovers and manages available connectors. In Phase 1 this wraps
+the existing sync functions. Future connectors simply subclass
+BaseConnector and are auto-discovered.
+"""
+
+import logging
+from connectors.base import BaseConnector
+
+logger = logging.getLogger("connectors.registry")
+
+# Registry of known connector classes
+_connector_classes: list[type[BaseConnector]] = []
+
+
+def register(cls: type[BaseConnector]):
+    """Register a connector class."""
+    if cls not in _connector_classes:
+        _connector_classes.append(cls)
+    return cls
+
+
+def discover_connectors() -> list[BaseConnector]:
+    """Discover and instantiate all registered connectors.
+
+    Imports connector modules to trigger registration, then
+    returns instances of all registered connector classes.
+    """
+    # Import connector modules to trigger @register decorators
+    try:
+        import connectors.github  # noqa: F401
+    except ImportError:
+        logger.warning("Could not import connectors.github")
+
+    try:
+        import connectors.gmail  # noqa: F401
+    except ImportError:
+        logger.warning("Could not import connectors.gmail")
+
+    try:
+        import connectors.notion  # noqa: F401
+    except ImportError:
+        logger.warning("Could not import connectors.notion")
+
+    return [cls() for cls in _connector_classes]
+
+
+def get_connector(name: str) -> BaseConnector | None:
+    """Return a connector instance by name."""
+    for connector in discover_connectors():
+        if connector.name.lower() == name.lower() or connector.slug == name.lower():
+            return connector
+    return None
+
+
+def list_connectors() -> list[str]:
+    """Return names of all registered connectors."""
+    return [cls.name for cls in _connector_classes]

core/__init__.py

@@ -0,0 +1 @@
+# core package

core/chunker.py

@@ -0,0 +1,136 @@
+import datetime
+from storage.db import (
+    get_all_repositories,
+    get_all_documents,
+    get_all_emails,
+    insert_document_chunk,
+    clear_document_chunks
+)
+
+def chunk_text(text: str, chunk_size: int = 800, overlap: int = 120) -> list:
+    if not text:
+        return []
+    chunks = []
+    start = 0
+    while start < len(text):
+        end = start + chunk_size
+        chunk = text[start:end]
+        chunks.append(chunk)
+        if end >= len(text):
+            break
+        start += chunk_size - overlap
+    return chunks
+
+def generate_and_save_chunks(repo_names=None):
+    from storage.db import get_connection
+    created_at = datetime.datetime.now().isoformat()
+    
+    conn = get_connection()
+    cursor = conn.cursor()
+    
+    if repo_names is None:
+        clear_document_chunks()
+        
+        # 1. Repositories (Metadata Chunks for identity search boosting)
+        repos = get_all_repositories()
+        for repo in repos:
+            repo_text = f"Repository: {repo['repo_name']}\nDescription: {repo['description'] or ''}\nLanguage: {repo['language'] or ''}"
+            # Insert repository metadata as a distinct source type to avoid mixing with document content
+            insert_document_chunk(
+                repository_name=repo["repo_name"],
+                document_name="metadata",
+                source_type="repository_metadata",
+                chunk_text=repo_text,
+                chunk_index=0,
+                created_at=created_at
+            )
+            
+        # 2. Repository Documents
+        docs = get_all_documents()
+        for doc in docs:
+            content = doc["content"] or ""
+            chunks = chunk_text(content)
+            for idx, chunk in enumerate(chunks):
+                insert_document_chunk(
+                    repository_name=doc["repo_name"],
+                    document_name=doc["file_name"],
+                    source_type="document",
+                    chunk_text=chunk,
+                    chunk_index=idx,
+                    created_at=created_at
+                )
+                
+        # 3. Emails
+        emails = get_all_emails()
+        for email in emails:
+            email_text = f"Subject: {email['subject']}\nFrom: {email['sender']}\nDate: {email['received_at']}\nContent: {email['snippet']}"
+            chunks = chunk_text(email_text)
+            for idx, chunk in enumerate(chunks):
+                insert_document_chunk(
+                    repository_name=None,
+                    document_name=email["subject"] or email["message_id"],
+                    source_type="email",
+                    chunk_text=chunk,
+                    chunk_index=idx,
+                    created_at=created_at
+                )
+    else:
+        # Incremental sync for specific repositories or emails
+        for r_name in repo_names:
+            if r_name == "__emails__":
+                # Delete existing email chunks
+                cursor.execute("DELETE FROM document_chunks WHERE source_type = 'email'")
+                conn.commit()
+                # Insert email chunks
+                emails = get_all_emails()
+                for email in emails:
+                    email_text = f"Subject: {email['subject']}\nFrom: {email['sender']}\nDate: {email['received_at']}\nContent: {email['snippet']}"
+                    chunks = chunk_text(email_text)
+                    for idx, chunk in enumerate(chunks):
+                        insert_document_chunk(
+                            repository_name=None,
+                            document_name=email["subject"] or email["message_id"],
+                            source_type="email",
+                            chunk_text=chunk,
+                            chunk_index=idx,
+                            created_at=created_at
+                        )
+            else:
+                # Delete existing metadata and doc chunks for this repo
+                cursor.execute(
+                    "DELETE FROM document_chunks WHERE repository_name = ? AND source_type IN ('document', 'repository_metadata')",
+                    (r_name,)
+                )
+                conn.commit()
+                
+                # Insert repo metadata chunk
+                cursor.execute("SELECT repo_name, description, language FROM repositories WHERE repo_name = ?", (r_name,))
+                repo_row = cursor.fetchone()
+                if repo_row:
+                    repo_text = f"Repository: {repo_row[0]}\nDescription: {repo_row[1] or ''}\nLanguage: {repo_row[2] or ''}"
+                    insert_document_chunk(
+                        repository_name=repo_row[0],
+                        document_name="metadata",
+                        source_type="repository_metadata",
+                        chunk_text=repo_text,
+                        chunk_index=0,
+                        created_at=created_at
+                    )
+                
+                # Insert repo document chunks
+                cursor.execute("SELECT repo_name, file_name, content FROM repository_documents WHERE repo_name = ?", (r_name,))
+                doc_rows = cursor.fetchall()
+                for d_row in doc_rows:
+                    content = d_row[2] or ""
+                    chunks = chunk_text(content)
+                    for idx, chunk in enumerate(chunks):
+                        insert_document_chunk(
+                            repository_name=d_row[0],
+                            document_name=d_row[1],
+                            source_type="document",
+                            chunk_text=chunk,
+                            chunk_index=idx,
+                            created_at=created_at
+                        )
+    conn.close()
+