footprinter-cli 1.0.0rc1__py3-none-any.whl

footprinter/ingest/folder_indexer.py

@@ -0,0 +1,226 @@
+"""
+Folder structure indexer for Footprinter.
+
+Scans ~/Work and ~/Personal to discover folder structure before file indexing.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+import sqlite3
+from typing import TYPE_CHECKING, Dict, List, Tuple
+
+from footprinter.utils.time import utc_now_iso
+
+if TYPE_CHECKING:
+    from footprinter.ingest.database import Database
+
+logger = logging.getLogger(__name__)
+
+# Home directory
+HOME = os.path.expanduser("~")
+
+
+class FolderIndexer:
+    """Indexes folder structure for Footprinter."""
+
+    # Directories to always skip (system/build caches)
+    SKIP_DIRS = {
+        "node_modules",
+        "__pycache__",
+        ".git",
+        ".venv",
+        "venv",
+        "site-packages",
+        ".next",
+        ".sfdx",
+        ".sf",
+        ".pytest_cache",
+        ".mypy_cache",
+        ".eggs",
+        ".tox",
+        ".nox",
+        ".cache",
+        "dist",
+        "build",
+    }
+
+    def __init__(self, config: Dict, db: Database):
+        """
+        Initialize folder scanner.
+
+        Args:
+            config: Configuration dictionary
+            db: Shared Database instance
+        """
+        self.config = config
+        self.db = db
+
+    def should_skip_dir(self, dir_name: str) -> bool:
+        """Check if directory should be skipped.
+
+        v3 Architecture (2026-01): Scan ALL folders including hidden ones.
+        Hidden folders are scanned so their files can be indexed with status='hidden'.
+
+        Only skip regeneratable build/cache directories (node_modules, venv, etc.)
+        """
+        # Skip known build/cache directories (regeneratable dependencies)
+        if dir_name.lower() in self.SKIP_DIRS:
+            return True
+
+        # NOTE: Hidden directories (starting with .) are NOT skipped
+        # They are scanned so their files can be indexed with status='hidden'
+        # Filter hidden folders in the Web UI, not at scan time
+
+        return False
+
+    def scan_folders(self, root_paths: List[str]) -> List[Dict]:
+        """
+        Scan folder structure starting from root paths.
+
+        Args:
+            root_paths: List of root paths to scan (e.g., ['~/Work', '~/Personal'])
+
+        Returns:
+            List of folder dictionaries
+        """
+        folders = []
+
+        for root_path in root_paths:
+            expanded_root = os.path.expanduser(root_path)
+            if not os.path.isdir(expanded_root):
+                logger.warning(f"Root path does not exist: {expanded_root}")
+                continue
+
+            logger.info(f"Scanning folders in {expanded_root}...")
+
+            for dirpath, dirnames, _ in os.walk(expanded_root):
+                # Filter out directories to skip
+                dirnames[:] = [d for d in dirnames if not self.should_skip_dir(d)]
+
+                # Get relative path from home
+                if dirpath.startswith(HOME):
+                    relative_path = dirpath[len(HOME) :]
+                else:
+                    relative_path = dirpath
+
+                # Get parent path
+                parent_path = os.path.dirname(dirpath)
+
+                folder = {
+                    "path": dirpath,
+                    "relative_path": relative_path,
+                    "name": os.path.basename(dirpath) or relative_path,
+                    "parent_path": parent_path if parent_path != dirpath else None,
+                    "scanned_at": utc_now_iso(),
+                }
+
+                folders.append(folder)
+
+        logger.info(f"Found {len(folders)} folders")
+        return folders
+
+    def save_folders(self, folders: List[Dict]) -> Tuple[int, int]:
+        """
+        Save folders to database.
+
+        Args:
+            folders: List of folder dictionaries
+
+        Returns:
+            Tuple of (inserted_count, updated_count)
+        """
+        cursor = self.db.conn.cursor()
+
+        inserted = 0
+        updated = 0
+
+        for folder in folders:
+            try:
+                # Try insert first
+                cursor.execute(
+                    """
+                    INSERT INTO folders
+                    (path, relative_path, name, parent_path, scanned_at,
+                     indexed_at, updated_at)
+                    VALUES (?, ?, ?, ?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
+                """,
+                    (
+                        folder["path"],
+                        folder["relative_path"],
+                        folder["name"],
+                        folder["parent_path"],
+                        folder["scanned_at"],
+                    ),
+                )
+                inserted += 1
+            except sqlite3.IntegrityError:
+                # Update existing
+                cursor.execute(
+                    """
+                    UPDATE folders
+                    SET relative_path = ?,
+                        name = ?,
+                        parent_path = ?,
+                        scanned_at = ?,
+                        updated_at = CURRENT_TIMESTAMP
+                    WHERE path = ?
+                """,
+                    (
+                        folder["relative_path"],
+                        folder["name"],
+                        folder["parent_path"],
+                        folder["scanned_at"],
+                        folder["path"],
+                    ),
+                )
+                updated += 1
+
+        self.db.conn.commit()
+
+        logger.info(f"Saved folders: {inserted} inserted, {updated} updated")
+        return inserted, updated
+
+    def get_folder_stats(self) -> Dict:
+        """Get statistics about indexed folders."""
+        cursor = self.db.conn.cursor()
+
+        cursor.execute("SELECT COUNT(*) FROM folders")
+        total = cursor.fetchone()[0]
+
+        return {"total_folders": total}
+
+
+def main():
+    """Run folder indexer from command line."""
+    from footprinter.ingest.database import Database
+    from footprinter.paths import get_db_path
+    from footprinter.source_registry import get_config
+
+    # Load config
+    config = get_config()
+
+    # Database
+    db = Database(str(get_db_path()))
+
+    # Create indexer
+    indexer = FolderIndexer(config, db)
+
+    # Scan folders
+    root_paths = config.get("directories", [])
+    if not root_paths:
+        raise ValueError("No directories configured. Add directories to config/config.yaml.")
+    folders = indexer.scan_folders(root_paths)
+
+    # Save to database
+    inserted, updated = indexer.save_folders(folders)
+
+    # Log stats
+    stats = indexer.get_folder_stats()
+    logger.info("Folder Scan Complete:")
+    logger.info(f"  Total folders: {stats['total_folders']}")
+
+
+if __name__ == "__main__":
+    main()

footprinter/ingest/full_content_extractor.py

@@ -0,0 +1,321 @@
+"""
+Full content extraction with chunking for semantic search.
+"""
+
+import logging
+from pathlib import Path
+from typing import Dict, List, Optional
+
+from .content_extractors import ContentExtractor
+
+logger = logging.getLogger(__name__)
+
+_pypdf_warned = False
+_docx_warned = False
+
+
+class FullContentExtractor(ContentExtractor):
+    """Extract full content from files with intelligent chunking."""
+
+    def __init__(
+        self,
+        chunk_size: int = 1000,
+        chunk_overlap: float = 0.15,
+        max_file_size_bytes: int = 50 * 1024 * 1024,
+        file_types: Optional[List[str]] = None,
+        exclude_patterns: Optional[List[str]] = None,
+    ):
+        """
+        Initialize full content extractor.
+
+        Args:
+            chunk_size: Size of each chunk in characters
+            chunk_overlap: Fractional overlap between chunks (0.0 to 1.0).
+                        Note: chunking.py uses absolute chars; this uses a fraction.
+            max_file_size_bytes: Maximum file size to read (0 = no limit)
+            file_types: Allowlist of file extensions (e.g. [".md", ".txt"]).
+                        None means all supported types are extracted.
+            exclude_patterns: fnmatch patterns for file paths to skip.
+        """
+        super().__init__(max_preview_length=1000)  # Keep small preview for DB
+        self.chunk_size = chunk_size
+        self.chunk_overlap = chunk_overlap
+        self.max_file_size_bytes = max_file_size_bytes
+        self.file_types = [t.lower() for t in file_types] if file_types is not None else None
+        self.exclude_patterns = exclude_patterns
+
+    @classmethod
+    def from_config(cls, config: dict) -> "FullContentExtractor":
+        """Build a FullContentExtractor from the application config dict.
+
+        Reads ``config["indexing"]["max_file_size_mb"]`` (default 0 = no
+        limit) and ``config["vectorization"]`` (chunk_size, chunk_overlap,
+        file_types, exclude_patterns).  Missing vectorization keys fall back
+        to constructor defaults.
+        """
+        max_mb = config.get("indexing", {}).get("max_file_size_mb", 0)
+        vec_config = config.get("vectorization", {})
+        vec_kwargs: dict = {}
+        if "chunk_size" in vec_config:
+            vec_kwargs["chunk_size"] = vec_config["chunk_size"]
+        if "chunk_overlap" in vec_config:
+            vec_kwargs["chunk_overlap"] = vec_config["chunk_overlap"]
+        if "file_types" in vec_config:
+            vec_kwargs["file_types"] = vec_config["file_types"]
+        if "exclude_patterns" in vec_config:
+            vec_kwargs["exclude_patterns"] = vec_config["exclude_patterns"]
+        return cls(
+            max_file_size_bytes=int(max_mb * 1024 * 1024),
+            **vec_kwargs,
+        )
+
+    def extract_with_chunking(self, file_path: Path) -> List[Dict[str, str]]:
+        """
+        Extract content and split into chunks if necessary.
+
+        Returns:
+            List of chunk dictionaries with 'content', 'chunk_index', 'total_chunks'
+        """
+        # Extract full content
+        full_content = self._extract_full_content(file_path)
+
+        if not full_content or len(full_content) == 0:
+            return []
+
+        # If content is small enough, return single chunk
+        if len(full_content) <= self.chunk_size:
+            return [{"content": full_content, "chunk_index": 0, "total_chunks": 1}]
+
+        # Split into chunks with overlap
+        chunks = []
+        overlap_size = int(self.chunk_size * self.chunk_overlap)
+
+        start = 0
+        chunk_index = 0
+
+        while start < len(full_content):
+            end = min(start + self.chunk_size, len(full_content))
+            chunk_text = full_content[start:end]
+
+            chunks.append(
+                {
+                    "content": chunk_text,
+                    "chunk_index": chunk_index,
+                    "total_chunks": 0,  # Will update after loop
+                }
+            )
+
+            # Move start forward, with overlap
+            if end >= len(full_content):
+                break
+            start = end - overlap_size
+            chunk_index += 1
+
+        # Update total_chunks count
+        total = len(chunks)
+        for chunk in chunks:
+            chunk["total_chunks"] = total
+
+        if total > 1:
+            logger.info(f"Split {file_path.name} into {total} chunks")
+
+        return chunks
+
+    def _extract_full_content(self, file_path: Path) -> Optional[str]:
+        """Extract full content from file.
+
+        Gates checked in order: file type allowlist, exclude patterns,
+        file size limit. Returns None if any gate rejects the file.
+        """
+        # File type allowlist gate (cheap — check before stat())
+        file_type = file_path.suffix.lower()
+        if self.file_types is not None and file_type not in self.file_types:
+            logger.debug("Skipping %s: extension %s not in file_types allowlist", file_path.name, file_type)
+            return None
+
+        # Exclude patterns gate (fnmatch against full absolute path)
+        if self.exclude_patterns:
+            from fnmatch import fnmatch
+
+            path_str = str(file_path)
+            if any(fnmatch(path_str, pat) for pat in self.exclude_patterns):
+                logger.debug("Skipping %s: matched exclude pattern", file_path.name)
+                return None
+
+        # File size guard
+        if self.max_file_size_bytes > 0:
+            try:
+                file_size = file_path.stat().st_size
+                if file_size > self.max_file_size_bytes:
+                    logger.warning(
+                        f"Skipping {file_path.name}: {file_size} bytes "
+                        f"exceeds content extraction limit of {self.max_file_size_bytes} bytes"
+                    )
+                    return None
+            except OSError:
+                pass  # stat failed — let the read attempt handle the error
+
+        try:
+            # Text-based files
+            if file_type in [
+                ".txt",
+                ".md",
+                ".py",
+                ".js",
+                ".json",
+                ".yaml",
+                ".yml",
+                ".html",
+                ".css",
+                ".jsx",
+                ".tsx",
+            ]:
+                return self._extract_full_text(file_path)
+
+            # Documents
+            elif file_type == ".pdf":
+                return self._extract_full_pdf(file_path)
+            elif file_type in [".docx", ".doc"]:
+                return self._extract_full_docx(file_path)
+
+            # Data files
+            elif file_type == ".csv":
+                return self._extract_csv_full(file_path)
+
+            # Other text-like formats
+            elif file_type in [
+                ".xml",
+                ".svg",
+                ".rst",
+                ".toml",
+                ".ini",
+                ".cfg",
+                ".conf",
+                ".sh",
+                ".bash",
+                ".zsh",
+                ".fish",
+                ".sql",
+                ".graphql",
+                ".proto",
+                ".ts",
+                ".vue",
+                ".svelte",
+                ".astro",
+                ".java",
+                ".kt",
+                ".scala",
+                ".go",
+                ".rs",
+                ".rb",
+                ".php",
+                ".c",
+                ".h",
+                ".cpp",
+                ".hpp",
+                ".cs",
+                ".swift",
+                ".m",
+                ".r",
+                ".jl",
+                ".lua",
+                ".pl",
+                ".pm",
+                ".tf",
+                ".hcl",
+                ".dockerfile",
+                ".log",
+                ".env",
+                ".gitignore",
+                ".editorconfig",
+                ".tex",
+                ".bib",
+                ".org",
+            ]:
+                return self._extract_full_text(file_path)
+
+            else:
+                # Skip binary/unknown files (images, video, audio, archives, etc.)
+                return None
+
+        except Exception as e:
+            logger.debug(f"Could not extract content from {file_path}: {e}")
+            return None
+
+    def _extract_full_text(self, file_path: Path) -> Optional[str]:
+        """Extract full text content."""
+        try:
+            with open(file_path, "r", encoding="utf-8", errors="ignore") as f:
+                return f.read()
+        except Exception as e:
+            logger.debug(f"Error reading text file {file_path}: {e}")
+            return None
+
+    def _extract_full_pdf(self, file_path: Path) -> Optional[str]:
+        """Extract full PDF content."""
+        try:
+            import pypdf
+
+            with open(file_path, "rb") as f:
+                reader = pypdf.PdfReader(f)
+
+                text = ""
+                for page in reader.pages:
+                    text += page.extract_text() + "\n"
+
+                return text
+
+        except ImportError:
+            global _pypdf_warned
+            if not _pypdf_warned:
+                logger.warning("pypdf not installed, skipping PDF extraction")
+                _pypdf_warned = True
+            return None
+        except Exception as e:
+            logger.debug(f"Error reading PDF {file_path}: {e}")
+            return None
+
+    def _extract_full_docx(self, file_path: Path) -> Optional[str]:
+        """Extract full DOCX content."""
+        try:
+            import docx
+
+            doc = docx.Document(file_path)
+
+            text = ""
+            for para in doc.paragraphs:
+                text += para.text + "\n"
+
+            return text
+
+        except ImportError:
+            global _docx_warned
+            if not _docx_warned:
+                logger.warning("python-docx not installed, skipping DOCX extraction")
+                _docx_warned = True
+            return None
+        except Exception as e:
+            logger.debug(f"Error reading DOCX {file_path}: {e}")
+            return None
+
+    def _extract_csv_full(self, file_path: Path) -> Optional[str]:
+        """Extract CSV content (headers + sample rows)."""
+        try:
+            import csv
+
+            with open(file_path, "r", encoding="utf-8", errors="ignore") as f:
+                reader = csv.reader(f)
+
+                lines = []
+                for i, row in enumerate(reader):
+                    lines.append(",".join(row))
+
+                    # Limit to reasonable size
+                    if i >= 1000:  # First 1000 rows
+                        break
+
+                return "\n".join(lines)
+
+        except Exception as e:
+            logger.debug(f"Error reading CSV {file_path}: {e}")
+            return None

footprinter/ingest/orchestrator.py

@@ -0,0 +1,112 @@
+"""Thin facade — coordinates pipeline pipes via delegation to extracted modules."""
+
+import logging
+from typing import Dict, List
+
+from footprinter.connectors import discover_connectors, get_connector_pipes, get_schema_specs
+from footprinter.ingest.pipe_runner import PipeRunner
+from footprinter.ingest.registry import (
+    CORE_PIPE_REGISTRY,
+    get_all_pipes,
+    get_pipelines,
+    get_refresh_pipes,
+)
+from footprinter.paths import get_config_path, get_db_path
+from footprinter.services.ingest_service import IngestService
+from footprinter.source_registry import get_config
+
+logger = logging.getLogger(__name__)
+
+
+class DataPipelineOrchestrator:
+    """Composition root — merges core + connector sources, delegates to PipeRunner."""
+
+    def __init__(self, config_path: str = None):
+        self.config = get_config(config_path)
+        self.config_path = config_path or str(get_config_path())
+        self.db = None
+        self.full_mode = False
+        from footprinter.source_registry import remote_accounts
+        self.remote_accounts = remote_accounts()
+        self._connectors = discover_connectors()
+        connector_pipes = get_connector_pipes(self._connectors)
+        self.adapter_registry = {**CORE_PIPE_REGISTRY, **connector_pipes}
+
+        # Build connector metadata for pipeline resolution and skip hints
+        connector_pipelines: dict[str, list[str]] = {}
+        connector_pipe_map: dict[str, str] = {}
+        for name, spec in self._connectors.items():
+            connector_pipelines[name] = list(spec.adapter_entries.keys())
+            for pipe in spec.pipes:
+                connector_pipe_map[pipe] = name
+            for pipe in spec.adapter_entries:
+                connector_pipe_map[pipe] = name
+
+        self.pipelines = get_pipelines(connector_pipes, connector_pipelines)
+        self.refresh_pipes = get_refresh_pipes(connector_pipes, connector_pipelines)
+        self.all_pipes = get_all_pipes(connector_pipes)
+
+        # Ensure DB schema exists (fresh installs need tables before pipes run)
+        from .database import Database
+        Database(str(get_db_path()), connector_specs=get_schema_specs(self._connectors)).close()
+        from .processing import ProcessingPipeline, run_access_resolution
+        self.processing = ProcessingPipeline()
+        self.processing.register(
+            "access_resolution",
+            runner=lambda db: run_access_resolution(db, full_mode=self.full_mode),
+        )
+        self.runner = PipeRunner(
+            processing=self.processing, get_db=self._get_db,
+            config=self.config, config_path=self.config_path,
+            adapter_registry=self.adapter_registry, pipelines=self.pipelines,
+            all_pipes=self.all_pipes, connector_pipe_map=connector_pipe_map,
+        )
+        self.ingest_service = IngestService(self._get_db().conn, get_db=self._get_db)
+
+    def _get_db(self):
+        if self.db is None:
+            from .database import Database
+            self.db = Database(str(get_db_path()), connector_specs=get_schema_specs(self._connectors))
+        return self.db
+
+    def run_pipe(self, pipe: str) -> Dict:
+        """Execute a single pipe by name."""
+        self.runner.full_mode = self.full_mode
+        mode = "full" if self.full_mode else "incremental"
+        return self.ingest_service.run_pipe(pipe, mode=mode, trigger="cli", runner=self.runner)
+
+    def run_pipeline(self, pipeline_name: str, on_pipe_start=None, on_pipe_end=None, on_progress=None) -> List[Dict]:
+        """Execute all pipes in a named pipeline."""
+        if pipeline_name not in self.runner.pipelines:
+            raise ValueError(f"Unknown pipeline: {pipeline_name}. Available: {', '.join(self.runner.pipelines.keys())}")
+        return self.run_pipes(
+            self.runner.pipelines[pipeline_name],
+            on_pipe_start=on_pipe_start, on_pipe_end=on_pipe_end, on_progress=on_progress,
+        )
+
+    def run_pipes(self, pipes: List[str], on_pipe_start=None, on_pipe_end=None, on_progress=None) -> List[Dict]:
+        """Execute a list of specific pipes in order."""
+        self.runner.full_mode = self.full_mode
+        mode = "full" if self.full_mode else "incremental"
+
+        def hook(pipe, on_progress=None):
+            return self.ingest_service.run_pipe(
+                pipe, mode=mode, trigger="cli", runner=self.runner, on_progress=on_progress,
+            )
+
+        return self.ingest_service.run_pipes(
+            pipes, runner=self.runner, full_mode=self.full_mode,
+            on_pipe_start=on_pipe_start, on_pipe_end=on_pipe_end,
+            on_progress=on_progress, pipe_hook=hook,
+        )
+
+    def get_status(self) -> Dict:
+        """Return current data counts and pipeline health."""
+        from footprinter.ingest.status import get_status
+        return get_status(str(get_db_path()))
+
+    def close(self):
+        """Close the database connection and release resources."""
+        if self.db:
+            self.db.close()
+            self.db = None