footprinter-cli 1.0.0rc1__py3-none-any.whl

footprinter/ingest/db/security.py

@@ -0,0 +1,6 @@
+"""Security and permission database methods.
+
+Permission and visibility tables (permission_policies, visibility_policies)
+and per-row columns (mcp_read, mcp_view) are defined in
+schema.py. Resolution logic lives in permissions.py and visibility.py.
+"""

footprinter/ingest/file_indexer.py

@@ -0,0 +1,223 @@
+"""
+File indexer that coordinates file scanning and content extraction.
+"""
+
+import logging
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Callable, Dict, Optional
+
+from footprinter.db import files as files_db
+from footprinter.source_registry import get_config
+
+from .content_extractors import ContentExtractor
+from .database import Database
+from .file_scanner import FileScanner
+
+logger = logging.getLogger(__name__)
+
+
+class FileIndexer:
+    """File indexer coordinating all indexing operations."""
+
+    def __init__(self, config_path: str = None, last_run: Optional[datetime] = None, db: Optional["Database"] = None):
+        """
+        Initialize the indexer.
+
+        Args:
+            config_path: Path to config YAML file (default: resolved via get_config_path())
+            last_run: Timestamp of last successful run. If set, only index files
+                modified after this time. None means full scan.
+            db: Optional shared Database handle. If None, creates its own.
+        """
+        self.config = get_config(config_path)
+        self.db = db if db is not None else Database()
+        self._owns_db = db is None
+        self.incremental = last_run is not None
+
+        if last_run:
+            logger.info(f"Incremental mode: indexing files modified since {last_run}")
+        else:
+            logger.info("Full scan mode (no last_run provided)")
+
+        self._vector_store = None  # lazy
+        self._full_extractor = None  # lazy
+
+        self.file_scanner = FileScanner(self.config, since_datetime=last_run)
+        self.content_extractor = ContentExtractor()
+
+    def index_files(
+        self,
+        relationship_maps: Optional[Dict[str, Any]] = None,
+        on_progress: Optional[Callable[[int], None]] = None,
+    ) -> dict:
+        """Index all files from configured directories to files table.
+
+        Args:
+            relationship_maps: Optional pre-built maps for in-memory
+                project/folder resolution. When provided, avoids per-row SQL.
+            on_progress: Optional callback fired with cumulative file count
+                after each file is processed (inserted + updated + skipped).
+
+        Returns:
+            Dict with keys: inserted, updated, skipped, errors
+        """
+        logger.info("Starting file indexing to files...")
+
+        inserted_count = 0
+        updated_count = 0
+        skipped_count = 0
+        error_count = 0
+        total_processed = 0
+        batch = []
+        batch_size = 1000  # Commit every 1000 files for performance
+        self._indexed_paths = set()  # Track all indexed paths for stale detection
+
+        for file_metadata in self.file_scanner.scan_all_directories():
+            try:
+                # Extract content preview (only when opt-in enabled)
+                if self.config.get("indexing", {}).get("content_snippets", False):
+                    file_path = Path(file_metadata["file_path"])
+                    content_preview = self.content_extractor.extract(file_path)
+                    file_metadata["content_preview"] = content_preview
+                else:
+                    file_metadata["content_preview"] = None
+
+                # Add to batch
+                batch.append(file_metadata)
+                self._indexed_paths.add(file_metadata["file_path"])
+
+                # Batch insert for performance
+                if len(batch) >= batch_size:
+                    bi, bu, bs = self._insert_batch(batch, relationship_maps)
+                    inserted_count += bi
+                    updated_count += bu
+                    skipped_count += bs
+                    batch = []
+                    logger.info(
+                        f"Progress: {inserted_count:,} inserted,"
+                        f" {updated_count:,} updated,"
+                        f" {skipped_count:,} skipped..."
+                    )
+
+            except Exception as e:  # Intentional broad catch: batch loop must not abort on single-item errors
+                logger.error(f"Error indexing file {file_metadata.get('file_path')}: {e}")
+                error_count += 1
+            finally:
+                total_processed += 1
+                if on_progress is not None:
+                    on_progress(total_processed)
+
+        # Insert remaining files
+        if batch:
+            bi, bu, bs = self._insert_batch(batch, relationship_maps)
+            inserted_count += bi
+            updated_count += bu
+            skipped_count += bs
+
+        # Mark stale files (no longer on disk)
+        # Only do this in full mode - incremental mode only scans modified files
+        if not self.incremental:
+            removed_ids = files_db.mark_removed_files(self.db.conn, self._indexed_paths)
+            if removed_ids:
+                store = self._get_vector_store()
+                if store:
+                    for file_id in removed_ids:
+                        try:
+                            store.delete_file(file_id)
+                        except Exception:  # Intentional broad catch: vector cleanup is best-effort
+                            logger.warning("Failed to delete vectors for removed file_id=%s", file_id, exc_info=True)
+                logger.info(f"Marked {len(removed_ids):,} files as stale (no longer on disk)")
+        else:
+            logger.info("Skipping stale detection in incremental mode")
+
+        logger.info(
+            f"File indexing complete: {inserted_count:,} inserted,"
+            f" {updated_count:,} updated,"
+            f" {skipped_count:,} skipped,"
+            f" {error_count:,} errors"
+        )
+        return {
+            "inserted": inserted_count,
+            "updated": updated_count,
+            "skipped": skipped_count,
+            "errors": error_count,
+        }
+
+    def _get_vector_store(self):
+        if self._vector_store is None:
+            try:
+                from footprinter.semantic.vector_store import VectorStore
+
+                self._vector_store = VectorStore.get_instance()
+            except Exception as e:  # Intentional broad catch: vector store is optional; any init failure disables it
+                logger.warning("Vector store unavailable: %s", e)
+                self._vector_store = False  # sentinel: don't retry
+        return self._vector_store if self._vector_store is not False else None
+
+    def _vectorize_file(self, file_id, file_path):
+        try:
+            from footprinter.semantic.vector_store import _file_vectorization_enabled
+        except ImportError:
+            return
+        if not _file_vectorization_enabled():
+            return
+        # Check per-record vectorize flag
+        row = self.db.conn.execute(
+            "SELECT COALESCE(json_extract(metadata, '$.vectorize'), 1) as vec FROM files WHERE id = ?",
+            (file_id,),
+        ).fetchone()
+        if row and row["vec"] == 0:
+            return
+        store = self._get_vector_store()
+        if not store:
+            return
+        try:
+            path = Path(file_path)
+            if not path.exists():
+                return
+            if self._full_extractor is None:
+                from .full_content_extractor import FullContentExtractor
+
+                self._full_extractor = FullContentExtractor.from_config(self.config)
+            chunks = self._full_extractor.extract_with_chunking(path)
+            if not chunks:
+                return
+            metadata = {"file_type": path.suffix.lower(), "file_name": path.name}
+            store.upsert_file(file_id, file_path, chunks, metadata)
+            self.db.conn.execute(
+                "UPDATE files SET vectorized_at = CURRENT_TIMESTAMP, vectorized_chunks = ? WHERE id = ?",
+                (len(chunks), file_id),
+            )
+        except Exception as e:  # Intentional broad catch: file vectorization is optional enhancement
+            logger.debug(f"Vectorization skipped for {file_path}: {e}")
+
+    def _insert_batch(
+        self,
+        batch,
+        relationship_maps: Optional[Dict[str, Any]] = None,
+    ) -> tuple:
+        """Insert a batch of files into files table.
+
+        Returns:
+            Tuple of (inserted, updated, skipped) counts
+        """
+        inserted = 0
+        updated = 0
+        skipped = 0
+        for file_metadata in batch:
+            result = files_db.insert_file(self.db.conn, file_metadata, relationship_maps=relationship_maps)
+            if result is None:
+                skipped += 1
+            else:
+                result_type, file_id = result
+                if result_type == "inserted":
+                    inserted += 1
+                else:
+                    updated += 1
+                self._vectorize_file(
+                    file_id,
+                    file_metadata.get("file_path") or file_metadata.get("path"),
+                )
+        self.db.conn.commit()
+        return inserted, updated, skipped

footprinter/ingest/file_scanner.py

@@ -0,0 +1,277 @@
+"""
+File system scanner with exclusion support.
+"""
+
+import logging
+import os
+import re
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Dict, Generator, List, Optional
+
+from ..utils.hash_utils import compute_md5, compute_sha256
+from ..utils.time import UTC_FMT
+
+logger = logging.getLogger(__name__)
+
+
+def _get_creation_time(stat_result: os.stat_result) -> float:
+    """Return the best available file creation timestamp.
+
+    On macOS (and Python 3.12+ on Linux with supported filesystems),
+    ``st_birthtime`` gives the true creation time. On older Linux,
+    ``st_ctime`` is the inode-change time (chmod, chown) — not creation —
+    so we fall back to ``st_mtime`` as the closest available proxy.
+    """
+    if hasattr(stat_result, "st_birthtime") and stat_result.st_birthtime > 0:
+        return stat_result.st_birthtime
+    return stat_result.st_mtime
+
+
+def _expand_home(pattern: str) -> str:
+    """Expand ~ to the current user's home directory in regex patterns.
+
+    Patterns starting with ^~/ have the ~ replaced with the regex-escaped
+    home directory path, making them platform-agnostic.
+    """
+    if pattern.startswith("^~/"):
+        home = re.escape(os.path.expanduser("~"))
+        return "^" + home + pattern[2:]
+    return pattern
+
+
+class FileScanner:
+    """File system scanner with configurable exclusion patterns."""
+
+    def __init__(self, config: Dict, since_datetime: Optional[datetime] = None):
+        """
+        Initialize file scanner.
+
+        Args:
+            config: Configuration dictionary
+            since_datetime: If provided, only scan files modified after this datetime
+                           (for incremental indexing)
+        """
+        self.config = config
+        self.since_datetime = since_datetime
+        self.always_exclusions = self._compile_always_exclusions()
+        self.sensitive_exclusions = self._compile_sensitive_exclusions()
+        self.supported_extensions = set(config.get("indexing", {}).get("supported_extensions", []))
+        # 0 = no size limit (matches config.example.yaml)
+        self.max_file_size = config.get("indexing", {}).get("max_file_size_mb", 0) * 1024 * 1024
+
+    def _compile_always_exclusions(self) -> List[re.Pattern]:
+        """Compile 'always' exclusion patterns (apply to all folders)."""
+        patterns = []
+        exclusions = self.config.get("exclusions", {})
+
+        for pattern in exclusions.get("always", []):
+            patterns.append(re.compile(_expand_home(pattern)))
+
+        return patterns
+
+    def _compile_sensitive_exclusions(self) -> List[re.Pattern]:
+        """Compile sensitive exclusion patterns (apply to all folders)."""
+        patterns = []
+        exclusions = self.config.get("exclusions", {})
+
+        for pattern in exclusions.get("sensitive", []):
+            patterns.append(re.compile(_expand_home(pattern)))
+
+        return patterns
+
+    def should_exclude(self, file_path: str) -> bool:
+        """
+        Check if file should be excluded based on patterns.
+
+        v3 Architecture (2026-01): Index ALL files in ~/Work and ~/Personal.
+        Hidden files are indexed with status='hidden', not excluded.
+
+        Only exclude:
+        - always: Regeneratable dependencies (node_modules, venv, .git internals)
+        - sensitive: Credentials and keys (.aws, .ssh, .kube)
+
+        REMOVED: client_hidden exclusions - hidden files are now indexed with
+        status='hidden' and filtered in the Web UI, not at scan time.
+        """
+        # Check 'always' exclusions (node_modules, venv, .git internals, etc.)
+        # These are regeneratable dependencies and system noise
+        for pattern in self.always_exclusions:
+            if pattern.search(file_path):
+                logger.debug(f"Excluding {file_path} (always pattern)")
+                return True
+
+        # Check sensitive exclusions (.aws, .ssh, .kube - credentials)
+        # These should never be indexed for security
+        for pattern in self.sensitive_exclusions:
+            if pattern.search(file_path):
+                logger.debug(f"Excluding {file_path} (sensitive pattern)")
+                return True
+
+        # NOTE: Hidden files (starting with .) are NOT excluded
+        # They are indexed with status='hidden' in the database
+
+        return False
+
+    def is_supported_file(self, file_path: Path) -> bool:
+        """Check if file type is supported."""
+        if not self.supported_extensions:
+            return True  # No filter = support all
+        return file_path.suffix.lower() in self.supported_extensions
+
+    def get_file_metadata(self, file_path: Path) -> Optional[Dict]:
+        """Extract file metadata."""
+        try:
+            stat = file_path.stat()
+
+            # Skip if too large (0 = no limit)
+            if self.max_file_size > 0 and stat.st_size > self.max_file_size:
+                logger.debug(f"Skipping {file_path} (too large: {stat.st_size} bytes)")
+                return None
+
+            file_path_str = str(file_path.absolute())
+
+            # Calculate both hashes:
+            # - SHA-256 for content deduplication
+            # - MD5 for Google Drive matching (Drive uses MD5)
+            sha256_hash = compute_sha256(file_path_str)
+            md5_hash = compute_md5(file_path_str)
+
+            return {
+                "file_path": file_path_str,
+                "file_name": file_path.name,
+                "file_type": file_path.suffix.lower() or "no_extension",
+                "file_size": stat.st_size,
+                "created_at": datetime.fromtimestamp(_get_creation_time(stat), tz=timezone.utc).strftime(UTC_FMT),
+                "modified_at": datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc).strftime(UTC_FMT),
+                "accessed_at": datetime.fromtimestamp(stat.st_atime, tz=timezone.utc).strftime(UTC_FMT),
+                "sha256_hash": sha256_hash,
+                "md5_hash": md5_hash,
+                "metadata": {
+                    "permissions": oct(stat.st_mode)[-3:],
+                },
+            }
+        except (OSError, OverflowError) as e:
+            logger.error(f"Error reading metadata for {file_path}: {e}")
+            return None
+
+    def scan_directory(self, directory: str) -> Generator[Dict, None, None]:
+        """
+        Scan directory and yield file metadata.
+
+        Yields file metadata dictionaries for indexing.
+        If since_datetime is set, only yields files modified after that time.
+        """
+        directory_path = Path(directory).expanduser().resolve()
+
+        if not directory_path.exists():
+            logger.error(f"Directory does not exist: {directory_path}")
+            return
+
+        if not directory_path.is_dir():
+            logger.error(f"Path is not a directory: {directory_path}")
+            return
+
+        if self.since_datetime:
+            logger.info(f"Scanning directory: {directory_path} (incremental since {self.since_datetime})")
+        else:
+            logger.info(f"Scanning directory: {directory_path}")
+
+        file_count = 0
+        excluded_count = 0
+        skipped_unchanged = 0
+        error_count = 0
+        seen_real_paths: set[str] = set()
+
+        try:
+            for root, dirs, files in os.walk(directory_path, followlinks=True):
+                # Check if current directory should be excluded
+                # Also check resolved path for directory symlinks
+                root_path = Path(root)
+                if self.should_exclude(root):
+                    dirs[:] = []  # Don't recurse into this directory
+                    excluded_count += len(files)
+                    continue
+                if root_path.is_symlink():
+                    resolved_root = str(root_path.resolve())
+                    if self.should_exclude(resolved_root) or resolved_root in seen_real_paths:
+                        dirs[:] = []
+                        excluded_count += len(files)
+                        continue
+                    seen_real_paths.add(resolved_root)
+
+                for file_name in files:
+                    file_path = Path(root) / file_name
+
+                    # Skip broken symlinks early (before stat() call)
+                    if file_path.is_symlink() and not file_path.exists():
+                        logger.debug(f"Skipping broken symlink: {file_path}")
+                        continue
+
+                    # Skip excluded files (check symlink path)
+                    if self.should_exclude(str(file_path)):
+                        excluded_count += 1
+                        continue
+
+                    # For symlinks, also check the resolved target against exclusions
+                    if file_path.is_symlink():
+                        real_path = str(file_path.resolve())
+                        if self.should_exclude(real_path):
+                            excluded_count += 1
+                            continue
+                    else:
+                        real_path = str(file_path.resolve())
+
+                    # Dedup: skip if we've already seen this real path
+                    if real_path in seen_real_paths:
+                        logger.debug(f"Skipping duplicate path: {file_path} -> {real_path}")
+                        continue
+                    seen_real_paths.add(real_path)
+
+                    # Skip unsupported file types
+                    if not self.is_supported_file(file_path):
+                        continue
+
+                    # For incremental indexing, skip files not modified since last run
+                    if self.since_datetime:
+                        try:
+                            mtime = datetime.fromtimestamp(file_path.stat().st_mtime, tz=timezone.utc)
+                            since = (
+                                self.since_datetime.replace(tzinfo=timezone.utc)
+                                if self.since_datetime.tzinfo is None
+                                else self.since_datetime
+                            )
+                            if mtime <= since:
+                                skipped_unchanged += 1
+                                continue
+                        except OSError:
+                            logger.debug("Could not check mtime for %s, processing anyway", file_path)
+
+                    # Get metadata
+                    metadata = self.get_file_metadata(file_path)
+                    if metadata:
+                        file_count += 1
+                        yield metadata
+                    else:
+                        error_count += 1
+
+        except OSError as e:
+            logger.error(f"Error scanning directory {directory_path}: {e}")
+            error_count += 1
+
+        if self.since_datetime:
+            logger.info(
+                f"Scan complete: {file_count} new/modified,"
+                f" {skipped_unchanged} unchanged,"
+                f" {excluded_count} excluded,"
+                f" {error_count} errors"
+            )
+        else:
+            logger.info(f"Scan complete: {file_count} files, {excluded_count} excluded, {error_count} errors")
+
+    def scan_all_directories(self) -> Generator[Dict, None, None]:
+        """Scan all configured directories."""
+        directories = self.config.get("directories", [])
+
+        for directory in directories:
+            yield from self.scan_directory(directory)