docsif 0.1.0__py3-none-any.whl

sif/indexing/parser.py

@@ -0,0 +1,192 @@
+"""Document parsing utilities."""
+
+from __future__ import annotations
+
+import hashlib
+import re
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+import frontmatter
+
+
+@dataclass
+class ParsedDocument:
+    """Result of parsing a document."""
+
+    path: str
+    content: str
+    title: str
+    metadata: dict[str, Any]
+    checksum: str
+    mtime: float
+
+
+class MarkdownParser:
+    """Parser for Markdown documents."""
+
+    def __init__(self) -> None:
+        self.title_pattern = re.compile(r"^#\s+(.+)$", re.MULTILINE)
+
+    def parse(self, file_path: Path) -> ParsedDocument:
+        """Parse a markdown file.
+
+        Args:
+            file_path: Path to the markdown file
+
+        Returns:
+            Parsed document
+        """
+        # Read file
+        content = file_path.read_text(encoding="utf-8")
+
+        # Get file stats
+        stat = file_path.stat()
+        mtime = stat.st_mtime
+
+        # Parse frontmatter
+        try:
+            post = frontmatter.loads(content)
+            body = post.content
+            metadata = dict(post.metadata)
+        except Exception:
+            body = content
+            metadata = {}
+
+        # Extract title
+        title = metadata.get("title", "")
+        if not title:
+            # Try to extract from first H1
+            match = self.title_pattern.search(body)
+            title = match.group(1).strip() if match else file_path.stem
+
+        # Calculate checksum
+        checksum = hashlib.sha256(content.encode()).hexdigest()
+
+        return ParsedDocument(
+            path=str(file_path.resolve()),
+            content=body,
+            title=title,
+            metadata=metadata,
+            checksum=checksum,
+            mtime=mtime,
+        )
+
+
+class TextParser:
+    """Parser for plain text documents."""
+
+    def parse(self, file_path: Path) -> ParsedDocument:
+        """Parse a text file."""
+        content = file_path.read_text(encoding="utf-8")
+        stat = file_path.stat()
+
+        # Use first line as title if not too long
+        lines = content.split("\n")
+        title = ""
+        for line in lines:
+            stripped = line.strip()
+            if stripped and len(stripped) < 100:
+                title = stripped
+                break
+
+        if not title:
+            title = file_path.stem
+
+        checksum = hashlib.sha256(content.encode()).hexdigest()
+
+        return ParsedDocument(
+            path=str(file_path.resolve()),
+            content=content,
+            title=title,
+            metadata={},
+            checksum=checksum,
+            mtime=stat.st_mtime,
+        )
+
+
+class CodeParser:
+    """Parser for code files."""
+
+    # File extensions to language mapping
+    LANGUAGE_MAP = {
+        ".py": "python",
+        ".js": "javascript",
+        ".ts": "typescript",
+        ".jsx": "jsx",
+        ".tsx": "tsx",
+        ".rs": "rust",
+        ".go": "go",
+        ".java": "java",
+        ".cpp": "cpp",
+        ".c": "c",
+        ".h": "c",
+        ".hpp": "cpp",
+        ".rb": "ruby",
+        ".php": "php",
+        ".swift": "swift",
+        ".kt": "kotlin",
+        ".scala": "scala",
+        ".r": "r",
+        ".sh": "bash",
+        ".yaml": "yaml",
+        ".yml": "yaml",
+        ".json": "json",
+        ".toml": "toml",
+        ".xml": "xml",
+        ".sql": "sql",
+    }
+
+    def parse(self, file_path: Path) -> ParsedDocument:
+        """Parse a code file."""
+        content = file_path.read_text(encoding="utf-8")
+        stat = file_path.stat()
+
+        # Detect language
+        ext = file_path.suffix.lower()
+        language = self.LANGUAGE_MAP.get(ext, "text")
+
+        # Try to extract title from comments
+        title = file_path.stem
+
+        # For Python, look for module docstring
+        if language == "python":
+            match = re.search(r'^[\'"]{3}(.+?)[\'"]{3}', content, re.MULTILINE | re.DOTALL)
+            if match:
+                docstring = match.group(1).strip()
+                first_line = docstring.split("\n")[0].strip()
+                if first_line:
+                    title = first_line[:100]
+
+        checksum = hashlib.sha256(content.encode()).hexdigest()
+
+        return ParsedDocument(
+            path=str(file_path.resolve()),
+            content=content,
+            title=title,
+            metadata={"language": language},
+            checksum=checksum,
+            mtime=stat.st_mtime,
+        )
+
+
+def create_parser(file_path: Path) -> MarkdownParser | TextParser | CodeParser | None:
+    """Create appropriate parser for a file.
+
+    Args:
+        file_path: Path to the file
+
+    Returns:
+        Parser instance or None
+    """
+    ext = file_path.suffix.lower()
+
+    if ext in [".md", ".markdown", ".mdx"]:
+        return MarkdownParser()
+    if ext in CodeParser.LANGUAGE_MAP:
+        return CodeParser()
+    if ext in [".txt", ".rst", ".adoc"]:
+        return TextParser()
+
+    return None

sif/indexing/scanner.py

@@ -0,0 +1,148 @@
+"""File scanning utilities."""
+
+from __future__ import annotations
+
+import fnmatch
+from dataclasses import dataclass, field
+from pathlib import Path
+
+
+@dataclass
+class ScanResult:
+    """Result of scanning a directory."""
+
+    files: list[Path] = field(default_factory=list)
+    scanned_dirs: set[Path] = field(default_factory=set)
+    file_count: int = 0
+
+    def __post_init__(self):
+        self.file_count = len(self.files)
+
+
+class FileScanner:
+    """Scanner for finding files matching patterns."""
+
+    DEFAULT_IGNORE_PATTERNS = [
+        "node_modules",
+        ".git",
+        ".svn",
+        ".hg",
+        "__pycache__",
+        ".pytest_cache",
+        ".mypy_cache",
+        ".tox",
+        ".venv",
+        "venv",
+        "env",
+        ".env",
+        "dist",
+        "build",
+        "*.egg-info",
+        ".DS_Store",
+        "Thumbs.db",
+        "*.swp",
+        "*.swo",
+        "*~",
+        ".idea",
+        ".vscode",
+    ]
+
+    def __init__(self) -> None:
+        pass
+
+    def scan(
+        self,
+        root_path: Path,
+        pattern: str = "**/*.md",
+        ignore_patterns: list[str] | None = None,
+    ) -> ScanResult:
+        """Scan a directory for files matching a pattern.
+
+        Args:
+            root_path: Root directory to scan
+            pattern: Glob pattern to match files
+            ignore_patterns: Patterns to ignore
+
+        Returns:
+            Scan result with matching files
+        """
+        root_path = Path(root_path).resolve()
+
+        # Combine default and custom ignore patterns
+        all_ignore_patterns = list(self.DEFAULT_IGNORE_PATTERNS)
+        if ignore_patterns:
+            all_ignore_patterns.extend(ignore_patterns)
+
+        result = ScanResult()
+        result.scanned_dirs.add(root_path)
+
+        # Use rglob for recursive matching
+        for file_path in root_path.rglob(pattern):
+            if not file_path.is_file():
+                continue
+
+            # Check if file should be ignored
+            if self._should_ignore(file_path, root_path, all_ignore_patterns):
+                continue
+
+            result.files.append(file_path)
+
+        result.file_count = len(result.files)
+        return result
+
+    def _should_ignore(
+        self,
+        file_path: Path,
+        root_path: Path,
+        ignore_patterns: list[str],
+    ) -> bool:
+        """Check if a file should be ignored.
+
+        Args:
+            file_path: Path to check
+            root_path: Root directory
+            ignore_patterns: Patterns to ignore
+
+        Returns:
+            True if file should be ignored
+        """
+        # Get relative path from root
+        try:
+            rel_path = file_path.relative_to(root_path)
+        except ValueError:
+            return False
+
+        # Check each part of the path against ignore patterns
+        for part in rel_path.parts:
+            for pattern in ignore_patterns:
+                if fnmatch.fnmatch(part, pattern):
+                    return True
+                if fnmatch.fnmatch(str(rel_path), pattern):
+                    return True
+
+        # Check filename
+        return any(fnmatch.fnmatch(file_path.name, pattern) for pattern in ignore_patterns)
+
+    def get_stats(self, scan_result: ScanResult) -> dict:
+        """Get statistics about a scan result.
+
+        Args:
+            scan_result: Scan result
+
+        Returns:
+            Statistics dictionary
+        """
+        total_size = sum(f.stat().st_size for f in scan_result.files if f.exists())
+
+        # Count by extension
+        extensions = {}
+        for f in scan_result.files:
+            ext = f.suffix.lower() or "(no extension)"
+            extensions[ext] = extensions.get(ext, 0) + 1
+
+        return {
+            "file_count": scan_result.file_count,
+            "total_size_bytes": total_size,
+            "total_size_mb": round(total_size / (1024 * 1024), 2),
+            "extensions": extensions,
+        }

sif/indexing/watcher.py

@@ -0,0 +1,156 @@
+"""File system watcher for auto-indexing."""
+
+from pathlib import Path
+
+from watchdog.events import FileSystemEvent, FileSystemEventHandler
+from watchdog.observers import Observer
+
+from sif.utils.logging import get_logger
+
+
+logger = get_logger(__name__)
+
+
+class IndexingEventHandler(FileSystemEventHandler):
+    """Event handler for file system changes."""
+
+    def __init__(
+        self,
+        collection_id: str,
+        indexer: "DocumentIndexer",  # noqa: F821
+        extensions: set[str] | None = None,
+    ) -> None:
+        """Initialize event handler.
+
+        Args:
+            collection_id: Collection ID
+            indexer: Document indexer
+            extensions: File extensions to watch
+        """
+        self._collection_id = collection_id
+        self._indexer = indexer
+        self._extensions = extensions or {".md", ".markdown"}
+
+    def on_created(self, event: FileSystemEvent) -> None:
+        """Handle file creation."""
+        if event.is_directory:
+            return
+
+        if self._should_handle(event.src_path):
+            logger.info(f"File created: {event.src_path}")
+            # Trigger incremental index
+
+    def on_modified(self, event: FileSystemEvent) -> None:
+        """Handle file modification."""
+        if event.is_directory:
+            return
+
+        if self._should_handle(event.src_path):
+            logger.info(f"File modified: {event.src_path}")
+            # Trigger incremental index
+
+    def on_deleted(self, event: FileSystemEvent) -> None:
+        """Handle file deletion."""
+        if event.is_directory:
+            return
+
+        if self._should_handle(event.src_path):
+            logger.info(f"File deleted: {event.src_path}")
+            # Trigger removal from index
+
+    def on_moved(self, event: FileSystemEvent) -> None:
+        """Handle file move/rename."""
+        if event.is_directory:
+            return
+
+        if self._should_handle(event.src_path):
+            logger.info(f"File moved: {event.src_path} -> {event.dest_path}")
+            # Trigger update
+
+    def _should_handle(self, path: str) -> bool:
+        """Check if a path should be handled.
+
+        Args:
+            path: File path
+
+        Returns:
+            True if path should be handled
+        """
+        return any(path.endswith(ext) for ext in self._extensions)
+
+
+class FileWatcher:
+    """File system watcher for auto-indexing.
+
+    Watches directories for changes and triggers incremental
+    indexing when files are created, modified, or deleted.
+    """
+
+    def __init__(
+        self,
+        indexer: "DocumentIndexer",  # noqa: F821
+        extensions: set[str] | None = None,
+    ) -> None:
+        """Initialize file watcher.
+
+        Args:
+            indexer: Document indexer
+            extensions: File extensions to watch
+        """
+        self._indexer = indexer
+        self._extensions = extensions or {".md", ".markdown"}
+        self._observer: Observer | None = None
+        self._handlers: dict[str, IndexingEventHandler] = {}
+
+    def start(self, collection_id: str, paths: list[str]) -> None:
+        """Start watching paths.
+
+        Args:
+            collection_id: Collection ID
+            paths: Paths to watch
+        """
+        logger.info(f"Starting file watcher for collection: {collection_id}")
+
+        self._observer = Observer()
+
+        handler = IndexingEventHandler(
+            collection_id=collection_id,
+            indexer=self._indexer,
+            extensions=self._extensions,
+        )
+
+        for path in paths:
+            if Path(path).exists():
+                self._observer.schedule(handler, path, recursive=True)
+                logger.info(f"Watching: {path}")
+
+        self._observer.start()
+        self._handlers[collection_id] = handler
+
+    def stop(self, collection_id: str | None = None) -> None:
+        """Stop watching.
+
+        Args:
+            collection_id: Collection to stop watching (None for all)
+        """
+        if collection_id and collection_id in self._handlers:
+            del self._handlers[collection_id]
+
+        if not collection_id or not self._handlers:
+            if self._observer:
+                self._observer.stop()
+                self._observer.join()
+                self._observer = None
+
+            logger.info("File watcher stopped")
+
+    def is_watching(self, collection_id: str) -> bool:
+        """Check if a collection is being watched.
+
+        Args:
+            collection_id: Collection ID
+
+        Returns:
+            True if collection is being watched
+        """
+        return collection_id in self._handlers