footprinter-cli 1.0.0rc1__py3-none-any.whl

footprinter/utils/hash_utils.py

@@ -0,0 +1,59 @@
+"""
+Hash computation utilities for Footprinter.
+
+Provides consistent hash computation for both local files and Google Drive matching.
+"""
+
+import hashlib
+import logging
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+
+def compute_md5(file_path: str) -> Optional[str]:
+    """
+    Compute MD5 hash matching Google Drive's md5Checksum.
+
+    Google Drive uses MD5 for file checksums, so this enables
+    hash-based matching between local files and Drive files.
+
+    Args:
+        file_path: Path to the file to hash
+
+    Returns:
+        32-character lowercase hex MD5 hash, or None on error
+    """
+    try:
+        hash_md5 = hashlib.md5()
+        with open(file_path, "rb") as f:
+            for chunk in iter(lambda: f.read(8192), b""):
+                hash_md5.update(chunk)
+        return hash_md5.hexdigest()
+    except (IOError, OSError) as e:
+        logger.debug(f"Could not compute MD5 for {file_path}: {e}")
+        return None
+
+
+def compute_sha256(file_path: str) -> Optional[str]:
+    """
+    Compute SHA-256 hash for content deduplication.
+
+    SHA-256 is used for content deduplication and integrity checks
+    within the local file system.
+
+    Args:
+        file_path: Path to the file to hash
+
+    Returns:
+        64-character lowercase hex SHA-256 hash, or None on error
+    """
+    try:
+        hash_sha256 = hashlib.sha256()
+        with open(file_path, "rb") as f:
+            for chunk in iter(lambda: f.read(8192), b""):
+                hash_sha256.update(chunk)
+        return hash_sha256.hexdigest()
+    except (IOError, OSError) as e:
+        logger.debug(f"Could not compute SHA-256 for {file_path}: {e}")
+        return None

footprinter/utils/logging_config.py

@@ -0,0 +1,68 @@
+"""Centralized logging configuration for Footprinter."""
+
+import logging
+import os
+import sys
+from pathlib import Path
+
+_configured = False
+
+
+def setup_logging(level=None):
+    """Configure the root logger. Only the first call takes effect.
+
+    Level resolution order:
+    1. Explicit ``level`` argument (if provided)
+    2. ``LOG_LEVEL`` environment variable (e.g. ``LOG_LEVEL=DEBUG``)
+    3. Falls back to INFO
+    """
+    global _configured
+    if _configured:
+        return
+    _configured = True
+
+    if level is None:
+        env_level = os.environ.get("LOG_LEVEL", "").upper()
+        level = getattr(logging, env_level, None) if env_level else None
+        if level is None:
+            level = logging.INFO
+
+    logging.basicConfig(
+        level=level,
+        format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+        stream=sys.stderr,
+    )
+
+
+def add_file_handler(log_path: Path, level: int = logging.DEBUG) -> logging.FileHandler:
+    """Add a file handler to the root logger for pipeline run logging.
+
+    Creates parent directories, sets a timestamped format, and suppresses
+    noisy schema migration logs. Returns the handler so it can be removed
+    after the run.
+    """
+    log_path.parent.mkdir(parents=True, exist_ok=True)
+
+    handler = logging.FileHandler(str(log_path))
+    handler.setLevel(level)
+    handler.setFormatter(logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s"))
+    logging.root.addHandler(handler)
+
+    # Ensure root logger level doesn't gate the file handler.
+    # --quiet suppresses Rich console output but NOT file logging.
+    if logging.root.level > level:
+        logging.root.setLevel(level)
+
+    # Suppress schema migration noise (INFO-level chatter on every run).
+    # Uses a handler filter instead of mutating the logger level so the
+    # suppression disappears when the handler is removed.
+    handler.addFilter(_schema_noise_filter)
+
+    return handler
+
+
+def _schema_noise_filter(record: logging.LogRecord) -> bool:
+    """Allow all records except low-level schema migration noise."""
+    if record.name.startswith("footprinter.ingest.db.schema"):
+        return record.levelno >= logging.WARNING
+    return True

footprinter/utils/mime.py

@@ -0,0 +1,30 @@
+"""MIME type to content type mapping.
+
+Used by both the orchestrator and Drive files adapter.
+"""
+
+
+def mime_to_content_type(mime_type: str) -> str:
+    """Convert MIME type to short content type string.
+
+    Returns a short label for known types (e.g. "pdf", "gdoc"),
+    or truncates the MIME subtype to 8 chars for unknown types.
+    Returns "unknown" for falsy input.
+    """
+    if not mime_type:
+        return "unknown"
+    mime_map = {
+        "application/pdf": "pdf",
+        "application/vnd.google-apps.document": "gdoc",
+        "application/vnd.google-apps.spreadsheet": "gsheet",
+        "application/vnd.google-apps.presentation": "gslides",
+        "application/vnd.openxmlformats-officedocument.wordprocessingml.document": "docx",
+        "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": "xlsx",
+        "application/vnd.openxmlformats-officedocument.presentationml.presentation": "pptx",
+        "text/plain": "txt",
+        "text/csv": "csv",
+        "image/jpeg": "jpg",
+        "image/png": "png",
+        "video/mp4": "mp4",
+    }
+    return mime_map.get(mime_type, mime_type.split("/")[-1][:8])

footprinter/utils/text.py

@@ -0,0 +1,6 @@
+import re
+
+
+def _make_slug(name: str) -> str:
+    """Convert a display name to a URL-safe slug."""
+    return re.sub(r"[^a-z0-9]+", "-", name.lower()).strip("-")

footprinter/utils/time.py

@@ -0,0 +1,11 @@
+"""Time utilities for consistent UTC timestamp handling."""
+
+from datetime import datetime, timezone
+
+UTC_FMT = "%Y-%m-%d %H:%M:%S"
+"""Timestamp format matching SQLite CURRENT_TIMESTAMP: YYYY-MM-DD HH:MM:SS (UTC)."""
+
+
+def utc_now_iso() -> str:
+    """Current UTC time in SQLite CURRENT_TIMESTAMP format."""
+    return datetime.now(timezone.utc).strftime(UTC_FMT)