cosma-backend 0.1.0__py3-none-any.whl

cosma_backend/utils/bundled.py

@@ -0,0 +1,24 @@
+import sys
+from importlib.resources import files
+from pathlib import Path
+
+def get_bundled_file(relative_path: str):
+    """Get path to bundled file, works in dev and production"""
+    if getattr(sys, 'frozen', False):
+        # Running in PyInstaller bundle
+        base_path = Path(sys._MEIPASS)  # type: ignore
+        return base_path / relative_path
+    else:
+        # Running in normal Python - use importlib.resources
+        # The package is 'backend' and schema.sql is in the package root
+        return files('backend').joinpath(relative_path)
+
+def get_bundled_file_text(relative_path: str) -> str:
+    """Get text content of bundled file, works in dev and production"""
+    if getattr(sys, 'frozen', False):
+        # Running in PyInstaller bundle
+        file_path = Path(sys._MEIPASS) / relative_path  # type: ignore
+        return file_path.read_text()
+    else:
+        # Running in normal Python - use importlib.resources
+        return files('backend').joinpath(relative_path).read_text()

cosma_backend/utils/pubsub.py

@@ -0,0 +1,31 @@
+# adapted from https://gist.github.com/appeltel/fd3ddeeed6c330c7208502462639d2c9
+
+import asyncio
+from contextlib import contextmanager
+import logging
+from typing import TypeVar, Generic
+
+logger = logging.getLogger(__name__)
+
+T = TypeVar('T')
+
+
+class Hub(Generic[T]):
+    subscriptions: set[asyncio.Queue[T]]
+    
+    def __init__(self):
+        self.subscriptions = set()
+
+    def publish(self, message: T):
+        for queue in self.subscriptions:
+            queue.put_nowait(message)
+
+
+@contextmanager
+def subscribe(hub: Hub[T]):
+    queue: asyncio.Queue[T] = asyncio.Queue()
+    hub.subscriptions.add(queue)
+    try:
+        yield queue
+    finally:
+        hub.subscriptions.remove(queue)

cosma_backend/utils/sse.py

@@ -0,0 +1,92 @@
+"""
+ServerSentEvent helper class for Quart SSE endpoints
+"""
+import json
+from typing import Optional, Any
+
+
+def sse_comment(text: str = "keepalive") -> str:
+    """
+    Create an SSE comment for keep-alive or debugging purposes.
+    
+    According to the SSE spec, lines starting with ':' are comments
+    and are ignored by clients. These are useful for:
+    - Keep-alive messages to prevent connection timeouts
+    - Debugging/logging without affecting the client
+    
+    Args:
+        text: Optional comment text (default: "keepalive")
+    
+    Returns:
+        Properly formatted SSE comment string with trailing double newline
+    
+    Example:
+        >>> sse_comment()
+        ': keepalive\\n\\n'
+        >>> sse_comment("heartbeat")
+        ': heartbeat\\n\\n'
+    """
+    return f": {text}\n\n"
+
+
+class ServerSentEvent:
+    """
+    Helper class to format Server-Sent Events according to the SSE specification.
+    
+    SSE format:
+        event: event_name
+        id: event_id
+        retry: retry_time
+        data: message_data
+        
+    """
+    
+    def __init__(
+        self,
+        data: Any,
+        event: Optional[str] = None,
+        id: Optional[str] = None,
+        retry: Optional[int] = None,
+    ):
+        """
+        Args:
+            data: The message data (will be JSON-encoded if not a string)
+            event: Optional event name for named events
+            id: Optional event ID (used for reconnection)
+            retry: Optional reconnection time in milliseconds
+        """
+        self.data = data
+        self.event = event
+        self.id = id
+        self.retry = retry
+    
+    def encode(self) -> str:
+        """
+        Encode the event in SSE format.
+        
+        Returns:
+            Properly formatted SSE string with trailing double newline
+        """
+        lines = []
+        
+        if self.event:
+            lines.append(f"event: {self.event}")
+        
+        if self.id:
+            lines.append(f"id: {self.id}")
+        
+        if self.retry:
+            lines.append(f"retry: {self.retry}")
+        
+        # Handle data - convert to JSON if not a string
+        if isinstance(self.data, str):
+            data_str = self.data
+        else:
+            data_str = json.dumps(self.data)
+        
+        # Support multi-line data
+        for line in data_str.splitlines():
+            lines.append(f"data: {line}")
+        
+        # SSE spec requires double newline at the end
+        return "\n".join(lines) + "\n\n"

cosma_backend/watcher/__init__.py

@@ -0,0 +1 @@
+from .watcher import Watcher as Watcher

cosma_backend/watcher/awatchdog.py

@@ -0,0 +1,80 @@
+import asyncio
+import functools
+from pathlib import Path
+from typing import Optional
+
+from watchdog.events import FileSystemEvent, FileSystemEventHandler
+from watchdog.observers import Observer
+from watchdog.observers.api import BaseObserver
+
+
+class _EventHandler(FileSystemEventHandler):
+    def __init__(self, queue: asyncio.Queue, loop: asyncio.AbstractEventLoop,
+                 *args, **kwargs):
+        self._loop = loop
+        self._queue = queue
+        super().__init__(*args, **kwargs)
+
+    def on_created(self, event: FileSystemEvent) -> None:
+        self._loop.call_soon_threadsafe(self._queue.put_nowait, event)
+    
+    def on_modified(self, event: FileSystemEvent) -> None:
+        self._loop.call_soon_threadsafe(self._queue.put_nowait, event)
+    
+    def on_deleted(self, event: FileSystemEvent) -> None:
+        self._loop.call_soon_threadsafe(self._queue.put_nowait, event)
+    
+    def on_moved(self, event: FileSystemEvent) -> None:
+        self._loop.call_soon_threadsafe(self._queue.put_nowait, event)
+
+
+class EventIterator:
+    def __init__(self, queue: asyncio.Queue,
+                 loop: Optional[asyncio.BaseEventLoop] = None):
+        self.queue = queue
+
+    def __aiter__(self):
+        return self
+
+    async def __anext__(self):
+        item = await self.queue.get()
+
+        if item is None:
+            raise StopAsyncIteration
+
+        return item
+
+
+def _watch(path: Path, queue: asyncio.Queue, loop: asyncio.AbstractEventLoop,
+          recursive: bool = False) -> BaseObserver:
+    """Watch a directory for changes."""
+
+    handler = _EventHandler(queue, loop)
+
+    observer = Observer()
+    observer.schedule(handler, str(path), recursive=recursive)
+    observer.start()
+    return observer
+    
+
+async def watch(path: Path, queue: asyncio.Queue, recursive: bool = False) -> BaseObserver:
+    loop = asyncio.get_running_loop()
+    partial = functools.partial(_watch, path=path, queue=queue, loop=loop, recursive=recursive)
+    return await asyncio.to_thread(partial)
+
+
+# async def consume(queue: asyncio.Queue) -> None:
+#     async for event in EventIterator(queue):
+#         print("Got an event!", event)
+
+
+# if __name__ == "__main__":
+#     loop = asyncio.get_event_loop()
+#     queue = asyncio.Queue(loop=loop)
+
+#     futures = [
+#         loop.run_in_executor(None, watch, Path("."), queue, loop, False),
+#         consume(queue),
+#     ]
+
+#     loop.run_until_complete(asyncio.gather(*futures))

cosma_backend/watcher/watcher.py

@@ -0,0 +1,257 @@
+import asyncio
+import datetime
+import logging
+from pathlib import Path
+from typing import Optional
+
+from watchdog.events import (
+    FileSystemEvent,
+    FileCreatedEvent,
+    FileModifiedEvent,
+    FileDeletedEvent,
+    FileMovedEvent,
+    DirCreatedEvent,
+    DirModifiedEvent,
+    DirDeletedEvent,
+    DirMovedEvent,
+)
+from watchdog.observers.api import BaseObserver
+
+from backend.db import Database
+from backend.logging import sm
+from backend.models import File
+from backend.models.watch import WatchedDirectory
+from backend.models.update import Update
+from backend.pipeline import Pipeline
+from backend.utils.pubsub import Hub
+from backend.watcher.awatchdog import watch
+
+logger = logging.getLogger(__name__)
+
+
+class WatcherJob:
+    pipeline: Pipeline
+    db: Database
+    watched_dir: WatchedDirectory
+    queue: asyncio.Queue[FileSystemEvent]
+    observer: Optional[BaseObserver]
+    task: Optional[asyncio.Task]
+    closed: bool
+    
+    def __init__(self, watched_dir: WatchedDirectory, pipeline: Pipeline, db: Database):
+        self.watched_dir = watched_dir
+        self.pipeline = pipeline
+        self.db = db
+        self.queue = asyncio.Queue()
+        self.observer = None
+        self.closed = False
+        self.task = None
+    
+    def _publish_update(self, update: Update):
+        """Publish an update through the pipeline's updates hub."""
+        if self.pipeline.updates_hub:
+            self.pipeline.updates_hub.publish(update)
+            
+    async def do_initial_processing(self):
+        await self.pipeline.process_directory(self.watched_dir.path)
+    
+    async def start(self):
+        logger.info(sm("Starting watchdog observer", watched_dir=self.watched_dir))
+        
+        # Publish watch started update
+        from backend.models.update import UpdateOpcode
+        self._publish_update(Update.create(
+            UpdateOpcode.WATCH_STARTED,
+            path=str(self.watched_dir.path),
+            recursive=self.watched_dir.recursive,
+            file_pattern=self.watched_dir.file_pattern
+        ))
+        
+        self.observer = await watch(self.watched_dir.path, self.queue, recursive=self.watched_dir.recursive)
+        self.task = asyncio.create_task(self.consumer_task())
+        asyncio.create_task(self.do_initial_processing())
+        
+    async def stop(self):
+        self.closed = True
+        if self.task is not None:
+            self.task.cancel()
+        if self.observer is not None:
+            self.observer.unschedule_all()
+        
+    async def consumer_task(self):
+        while not self.closed:
+            event = await self.queue.get()
+            
+            # Skip directory events - we only care about files
+            if isinstance(event, (DirCreatedEvent, DirModifiedEvent, DirDeletedEvent, DirMovedEvent)):
+                logger.debug(sm("Skipping directory event", event_type=type(event).__name__, path=event.src_path))
+                continue
+            
+            try:
+                # Handle different event types
+                if isinstance(event, FileDeletedEvent):
+                    logger.info(sm("File deleted", path=event.src_path))
+                    path = Path(str(event.src_path)).resolve()
+                    
+                    self._publish_update(Update.file_deleted(str(path)))
+                    await self.db.delete_file(str(path))
+                    
+                elif isinstance(event, FileMovedEvent):
+                    # Handle moved files as delete old + create new
+                    logger.info(sm("File moved", src=event.src_path, dest=event.dest_path))
+                    src_path = Path(str(event.src_path)).resolve()
+                    dest_path = Path(str(event.dest_path)).resolve()
+                    
+                    self._publish_update(Update.file_moved(str(src_path), str(dest_path)))
+                    await self.db.delete_file(str(src_path))
+                    
+                    # Check if destination file type is supported before processing
+                    dest_file = File.from_path(dest_path)
+                    if await self.pipeline.is_supported(dest_file):
+                        await self.pipeline.process_file(dest_file)
+                    else:
+                        logger.debug(sm("Skipping unsupported file type", path=str(dest_path)))
+                    
+                elif isinstance(event, (FileCreatedEvent, FileModifiedEvent)):
+                    # Handle created and modified files the same way - process them
+                    event_type = "created" if isinstance(event, FileCreatedEvent) else "modified"
+                    logger.info(sm(f"File {event_type}", path=event.src_path))
+                    path = Path(str(event.src_path)).resolve()
+                    
+                    # Publish file system event update
+                    if isinstance(event, FileCreatedEvent):
+                        self._publish_update(Update.file_created(str(path)))
+                    else:
+                        self._publish_update(Update.file_modified(str(path)))
+                    
+                    # Check if file type is supported before processing
+                    file = File.from_path(path)
+                    if await self.pipeline.is_supported(file):
+                        await self.pipeline.process_file(file)
+                    else:
+                        logger.debug(sm("Skipping unsupported file type", path=str(path)))
+                    
+                else:
+                    logger.warning(sm("Unknown event type", event_type=type(event).__name__, path=event.src_path))
+                    
+            except Exception as e:
+                logger.error(sm("Error processing file system event", event_type=type(event).__name__, path=event.src_path, error=e))
+    
+
+class Watcher:
+    jobs: set[WatcherJob]
+    
+    def __init__(
+        self,
+        db: Database,
+        pipeline: Pipeline,
+        updates_hub: Optional[Hub] = None,
+    ):
+        self.db = db
+        self.pipeline = pipeline
+        self.updates_hub = updates_hub
+        
+        self.jobs = set()
+
+    async def create_job(self, watched_dir: WatchedDirectory):
+        """
+        Create and start a watcher job for a watched directory.
+        
+        Args:
+            watched_dir: WatchedDirectory instance to watch
+        """
+        job = WatcherJob(watched_dir, self.pipeline, self.db)
+        self.jobs.add(job)
+        await job.start()
+
+    async def start_watching(self, path: str | Path, recursive: bool = True, file_pattern: Optional[str] = None):
+        """
+        Start watching a directory for file changes.
+        
+        Args:
+            path: Path to the directory to watch
+            recursive: Whether to watch subdirectories recursively
+            file_pattern: Optional glob pattern for filtering files (e.g., "*.pdf")
+            
+        Raises:
+            ValueError: If the directory is already being watched or a parent directory is being watched
+        """
+        # Create WatchedDirectory model
+        path = Path(path).resolve()
+        
+        # Check if this directory or any parent is already being watched
+        watched_dirs = await self.db.get_watched_directories(active_only=True)
+        
+        for existing_dir in watched_dirs:
+            # Check if exact path is already being watched
+            if existing_dir.path == path:
+                logger.warning(sm("Directory already being watched", path=str(path)))
+                raise ValueError(f"Directory '{path}' is already being watched")
+            
+            # Check if a parent directory is being watched with recursive=True
+            # A parent is watching this path if:
+            # 1. The parent is recursive
+            # 2. This path starts with the parent path
+            if existing_dir.recursive:
+                try:
+                    # Use relative_to to check if path is a subdirectory of existing_dir.path
+                    path.relative_to(existing_dir.path)
+                    # If we get here, path is a subdirectory of existing_dir.path
+                    logger.warning(sm("Parent directory already being watched", 
+                                     path=str(path), 
+                                     parent=str(existing_dir.path)))
+                    raise ValueError(f"Parent directory '{existing_dir.path}' is already watching '{path}' recursively")
+                except ValueError:
+                    # relative_to raises ValueError if path is not relative to existing_dir.path
+                    # This means it's not a subdirectory, so continue checking
+                    pass
+        
+        watched_dir = WatchedDirectory.from_path(path, recursive=recursive, file_pattern=file_pattern)
+        
+        # Add directory to watched_directories table in database
+        await self.db.add_watched_directory(watched_dir)
+        
+        # Create and start the watcher job
+        await self.create_job(watched_dir)
+
+    async def initialize_from_database(self):
+        """
+        Create jobs for all active watched directories from the database.
+        This should be called on startup to restore watching state.
+        """
+        logger.info(sm("Initializing watcher from database"))
+        
+        # Get all active watched directories
+        watched_dirs = await self.db.get_watched_directories(active_only=True)
+        
+        if not watched_dirs:
+            logger.info(sm("No watched directories found in database"))
+            return
+        
+        logger.info(sm("Found watched directories", count=len(watched_dirs)))
+        
+        # Create jobs for each watched directory
+        for watched_dir in watched_dirs:
+            # Check if path still exists
+            if not watched_dir.path.exists():
+                logger.warning(sm("Watched directory no longer exists", path=watched_dir.path_str, id=watched_dir.id))
+                continue
+            
+            if not watched_dir.path.is_dir():
+                logger.warning(sm("Watched path is not a directory", path=watched_dir.path_str, id=watched_dir.id))
+                continue
+            
+            logger.info(sm("Creating job for watched directory", 
+                          path=watched_dir.path_str, 
+                          id=watched_dir.id,
+                          recursive=watched_dir.recursive,
+                          file_pattern=watched_dir.file_pattern))
+            try:
+                await self.create_job(watched_dir)
+            except Exception as e:
+                logger.error(sm("Failed to create job for watched directory", 
+                               path=watched_dir.path_str, 
+                               id=watched_dir.id,
+                               error=str(e)))
+        
+        logger.info(sm("Watcher initialization complete", active_jobs=len(self.jobs)))

cosma_backend-0.1.0.dist-info/METADATA

@@ -0,0 +1,23 @@
+Metadata-Version: 2.3
+Name: cosma-backend
+Version: 0.1.0
+Summary: Flow backend
+Requires-Dist: asqlite>=2.0.0
+Requires-Dist: litellm>=1.77.5
+Requires-Dist: llama-cpp-python
+Requires-Dist: markitdown[docx,pdf,pptx]>=0.1.3
+Requires-Dist: ollama>=0.6.0
+Requires-Dist: quart>=0.20.0
+Requires-Dist: quart-schema>=0.22.0
+Requires-Dist: rich>=14.2.0
+Requires-Dist: sentence-transformers>=5.1.1
+Requires-Dist: sqlite-vec>=0.1.6
+Requires-Dist: tiktoken>=0.11.0
+Requires-Dist: uvicorn>=0.38.0
+Requires-Dist: watchdog>=6.0.0
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+
+# backend
+
+Cosma backend

cosma_backend-0.1.0.dist-info/RECORD

@@ -0,0 +1,39 @@
+cosma_backend/__init__.py,sha256=rUTO7_YoWZiLT396hMThilPjq58aDxa88d2vwrfVDS0,350
+cosma_backend/__main__.py,sha256=XCrM3GNtyOO3WoGWi9q1orJ7YZ1ZFtyIvS9uDHjvTV8,60
+cosma_backend/api/__init__.py,sha256=e7hpuncNrF5oRoMIWdQ5qz2OnV-GJzF0nUSYnYQdc9g,906
+cosma_backend/api/files.py,sha256=rgL4gn3VWfTLE2_ZeYJRGNbiBeXhCt6QxAWIRluhcVM,4041
+cosma_backend/api/index.py,sha256=I1GBhgYU_7AUAOndtgsF0mUIJ_d_cBgpIt_iSeX4kUQ,3049
+cosma_backend/api/models.py,sha256=ofqYTBdNpJ5PEbFngoF0Fp-BgBjiS_xKRWO15Ir4Y9A,616
+cosma_backend/api/search.py,sha256=AarPHBEMt4VuufBvRods40vSM9XS24SxcBkbTTY5Ic8,4421
+cosma_backend/api/status.py,sha256=zYbqdSmDaYWe0mw8ImZh9VKb3HzmKLw6acTPB7VlMf4,680
+cosma_backend/api/updates.py,sha256=NEfr-EaUlP0E_Al6mAyoz9a-w6N9D_edIR0wz_DrfqM,2305
+cosma_backend/api/watch.py,sha256=R3jic9QlNwJftBf_Wdia4KY_xyjhQxCGQUtbwBstCvk,4189
+cosma_backend/app.py,sha256=pJc1VgQizKywxS2La2rXM1veEIo2WgGzaa0f1-1WfJM,5267
+cosma_backend/db/__init__.py,sha256=qjJ-hkOUO4wu2XLs83-VOfiC7KmERJAfJTH50SOf5b4,84
+cosma_backend/db/database.py,sha256=DmifOHuNooXc38JjTLR485tnetuMl5Dhtv_nBFbHyrg,23388
+cosma_backend/discoverer/__init__.py,sha256=LJiZ0WZxuh4vTRP2q0fOLR0MY-y5NtpQnYj-UN9vHDM,49
+cosma_backend/discoverer/discoverer.py,sha256=PY7y_6E-zUY5KuDo_OmiPx5ok7m8GhuazUA_RObGcJw,900
+cosma_backend/embedder/__init__.py,sha256=wah-YAXJlU0muzMcQEdsEss6L1uxr8ul1CzDoGr6gIA,51
+cosma_backend/embedder/embedder.py,sha256=rim5_u0vDR2_UPukAgVqjFo5YOOtbp5MHX9JjkSFE4w,23894
+cosma_backend/logging.py,sha256=UdzeCANJ8tp9ewi0wsVJnsNasCXHy36sUKxJ5gJofQw,2172
+cosma_backend/models/__init__.py,sha256=0zOKcsF38_0w2R7vqa6vegltk0yE719a6zhYIRkMPho,144
+cosma_backend/models/file.py,sha256=nW2S5F-UDZMTi1aAFqzaC5qXmYro5T98oksyfGTEJRg,5655
+cosma_backend/models/status.py,sha256=wcnkM954LUfHhz3MCCb2gr5qmbFG2Wady7VbZRfZ9w4,135
+cosma_backend/models/update.py,sha256=rNrS9s75g8xAf0YMZXPTXT4ynI5fnWiHL6CABowbYyQ,7969
+cosma_backend/models/watch.py,sha256=JUlBYzrLQKmA02DhCCg7Q3k4Kjs_7hsPrfzCsIYEdME,4233
+cosma_backend/pipeline/__init__.py,sha256=0bzogjLkCVWexfdymwVW8QfOBFn02hF-quWFBR5mPQw,98
+cosma_backend/pipeline/pipeline.py,sha256=-_EiidzaGNSwOL69Uw3BOr4N3vKmjn8ZzyHyoacl8VE,8460
+cosma_backend/schema.sql,sha256=gpnJpJdhwGzq8c9ZgED0FbkecJE4Me0bjcJLfo_Lm6g,10817
+cosma_backend/searcher/__init__.py,sha256=Cb_GU3h7QjGC4I_gRkSlY7tT4eH-F4fectoj5YNTrj8,55
+cosma_backend/searcher/searcher.py,sha256=bmxzlyhLzsQqvDuYX3QuWdRZJlxmJT_Jn-vo118y_Nk,14877
+cosma_backend/summarizer/__init__.py,sha256=8Dz2YfNbfd4CpBkW-iwftprQDtXqgj71jt5o-4g62RE,993
+cosma_backend/summarizer/summarizer.py,sha256=qXWl3dpd95vgzxYthUHNnkiE3qCZm4-AkASC4W9bC3Q,45069
+cosma_backend/utils/bundled.py,sha256=hSW-RdSvVxMfBiA76E2G5N3u5h791G02bjpE3hrEMP0,1007
+cosma_backend/utils/pubsub.py,sha256=Prvx9MR0Eht7n40KqsjEL5yLthgkp88YsuBfciIGowI,694
+cosma_backend/utils/sse.py,sha256=mGrTMtNK-vD9B72drKft6NGjdzRvD1lEP11lHmY2-m4,2493
+cosma_backend/watcher/__init__.py,sha256=pXE-xQsBGV9Fp5ceUs6n-nEM3fkKLV4t0ONo0vmPlKQ,40
+cosma_backend/watcher/awatchdog.py,sha256=LT_6BzTg5aUIhDlgwNN1doSKrN6xQr76yUl4TPpHU3o,2444
+cosma_backend/watcher/watcher.py,sha256=RyqLweQ8EnUCt0kqJlezEwxN4TdJDC7qlld8eBXISDs,10873
+cosma_backend-0.1.0.dist-info/WHEEL,sha256=DpNsHFUm_gffZe1FgzmqwuqiuPC6Y-uBCzibcJcdupM,78
+cosma_backend-0.1.0.dist-info/METADATA,sha256=EgYuhbC3T-iPSTrBE-IgHjnEXVIZnkG2bY35IqjaEFg,603
+cosma_backend-0.1.0.dist-info/RECORD,,

cosma_backend-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: uv 0.9.8
+Root-Is-Purelib: true
+Tag: py3-none-any