mfcli 0.2.1__py3-none-any.whl

mfcli/utils/files.py

@@ -0,0 +1,16 @@
+import os
+import re
+from pathlib import Path
+from typing import Literal
+
+AppDataFileTypes = Literal['datasheets', 'files']
+
+
+def file_access_check(file_path: Path) -> bool:
+    if not os.path.exists(file_path) or not os.access(file_path, os.R_OK):
+        return False
+    return True
+
+
+def is_text_mime_type(mime_type: str) -> bool:
+    return bool(re.match(r"^text/.+$", mime_type))

mfcli/utils/http_requests.py

@@ -0,0 +1,54 @@
+import requests
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
+
+
+def get_retry_session(
+        retries: int = 3,
+        backoff_factor: float = 0.5,
+        status_forcelist: tuple = (500, 502, 503, 504),
+        allowed_methods: tuple = ("GET", "POST", "PUT", "DELETE", "PATCH")
+) -> requests.Session:
+    """Create a requests session with retry logic."""
+    session = requests.Session()
+    retry = Retry(
+        total=retries,
+        read=retries,
+        connect=retries,
+        backoff_factor=backoff_factor,
+        status_forcelist=status_forcelist,
+        allowed_methods=allowed_methods,
+        raise_on_status=False,
+    )
+    adapter = HTTPAdapter(max_retries=retry)
+    session.mount("http://", adapter)
+    session.mount("https://", adapter)
+    return session
+
+
+def http_request(
+        method: str,
+        url: str,
+        retries: int = 3,
+        timeout: int = 10,
+        **kwargs
+) -> requests.Response:
+    """
+    Perform an HTTP request with automatic retry logic.
+
+    :param method: HTTP method (GET, POST, PUT, DELETE, etc.)
+    :param url: URL to request
+    :param retries: Number of retry attempts
+    :param timeout: Timeout per request in seconds
+    :param kwargs: Any requests.request() parameters (headers, data, json, etc.)
+    :return: requests.Response object
+    """
+    session = get_retry_session(retries=retries)
+    try:
+        response = session.request(method.upper(), url, allow_redirects=True, timeout=timeout, **kwargs)
+        response.raise_for_status()
+        return response
+    except requests.exceptions.RequestException:
+        raise
+    finally:
+        session.close()

mfcli/utils/kb_lister.py

@@ -0,0 +1,89 @@
+from typing import Dict, List, Set
+from mfcli.client.chroma_db import get_chromadb_client_for_project_name
+from mfcli.utils.logger import get_logger
+from mfcli.utils.orm import Session
+
+logger = get_logger(__name__)
+
+
+def list_vectorized_files(project_name: str) -> Dict[str, List[str]]:
+    """
+    List all files that have been vectorized into the ChromaDB database for a project.
+    
+    Args:
+        project_name: Name of the project
+        
+    Returns:
+        Dictionary mapping purpose to list of file names
+    """
+    try:
+        with Session() as db:
+            chroma_client = get_chromadb_client_for_project_name(db, project_name)
+            
+        # Get all documents from the collection
+        collection = chroma_client._collection
+        results = collection.get()
+        
+        if not results or not results.get('metadatas'):
+            logger.info("No vectorized files found in the knowledge base")
+            return {}
+        
+        # Group files by purpose
+        files_by_purpose: Dict[str, Set[str]] = {}
+        
+        for metadata in results['metadatas']:
+            if metadata:
+                file_name = metadata.get('file_name', 'Unknown')
+                purpose = metadata.get('purpose', 'unknown')
+                
+                if purpose not in files_by_purpose:
+                    files_by_purpose[purpose] = set()
+                files_by_purpose[purpose].add(file_name)
+        
+        # Convert sets to sorted lists for consistent output
+        result = {
+            purpose: sorted(list(files)) 
+            for purpose, files in files_by_purpose.items()
+        }
+        
+        return result
+        
+    except Exception as e:
+        logger.error(f"Failed to list vectorized files for project: {project_name}")
+        logger.exception(e)
+        raise
+
+
+def print_vectorized_files(project_name: str) -> None:
+    """
+    Print all vectorized files in a formatted manner.
+    
+    Args:
+        project_name: Name of the project
+    """
+    try:
+        files_by_purpose = list_vectorized_files(project_name)
+        
+        if not files_by_purpose:
+            print(f"\nNo files have been vectorized for project '{project_name}' yet.")
+            return
+        
+        print(f"\nVectorized files in knowledge base for project '{project_name}':")
+        print("=" * 70)
+        
+        total_files = 0
+        for purpose in sorted(files_by_purpose.keys()):
+            files = files_by_purpose[purpose]
+            print(f"\n{purpose.upper()} ({len(files)} file{'s' if len(files) != 1 else ''}):")
+            print("-" * 70)
+            for file_name in files:
+                print(f"  • {file_name}")
+            total_files += len(files)
+        
+        print("\n" + "=" * 70)
+        print(f"Total: {total_files} unique file{'s' if total_files != 1 else ''} vectorized\n")
+        
+    except Exception as e:
+        logger.error(f"Failed to print vectorized files for project: {project_name}")
+        print(f"\nError: Failed to list vectorized files. Check logs for details.")
+        raise

mfcli/utils/kb_remover.py

@@ -0,0 +1,173 @@
+import json
+from pathlib import Path
+from typing import List
+from mfcli.client.chroma_db import get_chromadb_client_for_project_name
+from mfcli.models.file_docket import FileDocket
+from mfcli.utils.directory_manager import app_dirs
+from mfcli.utils.logger import get_logger
+from mfcli.utils.orm import Session
+
+logger = get_logger(__name__)
+
+
+def _remove_from_file_docket(matching_files: set) -> int:
+    """
+    Remove files from the file_docket.json based on file names.
+    
+    Args:
+        matching_files: Set of file names to remove
+        
+    Returns:
+        Number of entries removed from docket
+    """
+    if not app_dirs.file_docket_path or not app_dirs.file_docket_path.exists():
+        logger.warning("file_docket.json not found, skipping docket removal")
+        return 0
+    
+    try:
+        # Load existing file docket
+        docket = FileDocket()
+        docket.load_from_json(app_dirs.file_docket_path)
+        
+        # Track how many entries we remove
+        removed_count = 0
+        
+        # Find and remove matching entries
+        entries_to_remove = []
+        for entry in docket._docket.entries:
+            if entry.name in matching_files:
+                entries_to_remove.append(entry)
+        
+        for entry in entries_to_remove:
+            docket.remove(entry)
+            removed_count += 1
+            logger.debug(f"Removed from docket: {entry.name}")
+        
+        # Save updated docket back to file
+        if removed_count > 0:
+            json_data = json.dumps(docket.get_entries(), indent=2)
+            with open(app_dirs.file_docket_path, "w") as f:
+                f.write(json_data)
+            logger.info(f"Updated file_docket.json, removed {removed_count} entries")
+        
+        return removed_count
+        
+    except Exception as e:
+        logger.error(f"Failed to remove files from file_docket: {e}")
+        logger.exception(e)
+        return 0
+
+
+def remove_files_from_kb(project_name: str, filename_pattern: str, confirm: bool = True) -> int:
+    """
+    Remove files from the ChromaDB knowledge base that match the given filename pattern.
+    
+    Args:
+        project_name: Name of the project
+        filename_pattern: Full or partial filename to match (case-insensitive)
+        confirm: If True, ask for confirmation before deleting
+        
+    Returns:
+        Number of chunks deleted
+    """
+    try:
+        with Session() as db:
+            chroma_client = get_chromadb_client_for_project_name(db, project_name)
+        
+        # Get all documents from the collection
+        collection = chroma_client._collection
+        results = collection.get()
+        
+        if not results or not results.get('metadatas'):
+            logger.info("No files found in the knowledge base")
+            print("No files found in the knowledge base.")
+            return 0
+        
+        # Find matching file chunks
+        matching_ids: List[str] = []
+        matching_files: set = set()
+        
+        for idx, metadata in enumerate(results['metadatas']):
+            if metadata:
+                file_name = metadata.get('file_name', '')
+                # Case-insensitive partial match
+                if filename_pattern.lower() in file_name.lower():
+                    matching_ids.append(results['ids'][idx])
+                    matching_files.add(file_name)
+        
+        if not matching_ids:
+            print(f"\nNo files matching '{filename_pattern}' found in the knowledge base.")
+            return 0
+        
+        # Display matching files
+        print(f"\nFound {len(matching_files)} file(s) matching '{filename_pattern}':")
+        for file_name in sorted(matching_files):
+            print(f"  • {file_name}")
+        print(f"\nTotal chunks to delete: {len(matching_ids)}")
+        
+        # Confirm deletion
+        if confirm:
+            response = input("\nAre you sure you want to delete these files? (yes/no): ")
+            if response.lower() not in ['yes', 'y']:
+                print("Deletion cancelled.")
+                return 0
+        
+        # Delete the chunks from ChromaDB
+        collection.delete(ids=matching_ids)
+        
+        # Remove files from file_docket
+        removed_from_docket = _remove_from_file_docket(matching_files)
+        
+        print(f"\nSuccessfully deleted {len(matching_ids)} chunks from {len(matching_files)} file(s).")
+        if removed_from_docket > 0:
+            print(f"Removed {removed_from_docket} file(s) from file_docket.json")
+        logger.info(f"Deleted {len(matching_ids)} chunks from {len(matching_files)} files matching '{filename_pattern}'")
+        logger.info(f"Removed {removed_from_docket} entries from file_docket")
+        
+        return len(matching_ids)
+        
+    except Exception as e:
+        logger.error(f"Failed to remove files from knowledge base for project: {project_name}")
+        logger.exception(e)
+        print(f"\nError: Failed to remove files. Check logs for details.")
+        raise
+
+
+def list_matching_files(project_name: str, filename_pattern: str) -> List[str]:
+    """
+    List files that match the given filename pattern without deleting them.
+    
+    Args:
+        project_name: Name of the project
+        filename_pattern: Full or partial filename to match (case-insensitive)
+        
+    Returns:
+        List of matching filenames
+    """
+    try:
+        with Session() as db:
+            chroma_client = get_chromadb_client_for_project_name(db, project_name)
+        
+        # Get all documents from the collection
+        collection = chroma_client._collection
+        results = collection.get()
+        
+        if not results or not results.get('metadatas'):
+            return []
+        
+        # Find matching files
+        matching_files: set = set()
+        
+        for metadata in results['metadatas']:
+            if metadata:
+                file_name = metadata.get('file_name', '')
+                # Case-insensitive partial match
+                if filename_pattern.lower() in file_name.lower():
+                    matching_files.add(file_name)
+        
+        return sorted(list(matching_files))
+        
+    except Exception as e:
+        logger.error(f"Failed to list matching files for project: {project_name}")
+        logger.exception(e)
+        return []

mfcli/utils/logger.py

@@ -0,0 +1,28 @@
+import logging
+import sys
+
+from mfcli.utils.config import get_config
+
+
+def setup_logging():
+    config = get_config()
+    formatter = logging.Formatter(
+        fmt="%(asctime)s [%(levelname)s] %(name)s:%(funcName)s:%(lineno)d: %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S"
+    )
+
+    handler = logging.StreamHandler(sys.stdout)
+    handler.setFormatter(formatter)
+
+    logger = logging.getLogger()
+    logger.setLevel(config.log_level)
+    logger.addHandler(handler)
+    logger.propagate = False
+
+    logging.getLogger("google_genai").setLevel(logging.ERROR)
+    logging.getLogger("httpx").setLevel(logging.ERROR)
+    logging.getLogger("docling").setLevel(logging.ERROR)
+
+
+def get_logger(name=None):
+    return logging.getLogger(name or __name__)