mfcli 0.2.0__py3-none-any.whl

mfcli/utils/directory_manager.py

@@ -0,0 +1,96 @@
+import os
+import sys
+from pathlib import Path
+
+
+class DirectoryManager:
+    _instance = None
+
+    def __new__(cls):
+        if cls._instance is None:
+            cls._instance = super().__new__(cls)
+            cls._instance._initialized = False
+        return cls._instance
+
+    def __init__(self):
+        if self._initialized:
+            return
+        # OS-specific base appdata location
+        if os.name == "nt":
+            app_data_base = Path(os.getenv("LOCALAPPDATA", os.getenv("APPDATA")))
+        elif sys.platform == "darwin":
+            app_data_base = Path.home() / "Library" / "Application Support"
+        else:
+            app_data_base = Path.home() / ".local" / "share"
+
+        self.home_dir: Path = Path(os.path.expanduser("~")) / "Multifactor"
+        self.env_file_path: Path = self.home_dir / ".env"
+
+        # User app directories
+        self.app_data_dir: Path = app_data_base / "Multifactor"
+        self.chroma_db_dir: Path = self.app_data_dir / "chromadb"
+
+        self.app_data_dir.mkdir(exist_ok=True, parents=True)
+        self.chroma_db_dir.mkdir(exist_ok=True, parents=True)
+        self.home_dir.mkdir(exist_ok=True, parents=True)
+
+        # Repo dirs
+        self.root_dir: Path | None = None
+        self.agent_instructions_dir: Path | None = None
+        self.data_sheets_dir: Path | None = None
+        self.fw_tasks_dir: Path | None = None
+        self.generated_files_dir: Path | None = None
+        self.cheat_sheets_dir: Path | None = None
+        self.reqs_dir: Path | None = None
+        self.pdf_parts_dir: Path | None = None
+        self.metadata_dir: Path | None = None
+        self.config_file_path: Path | None = None
+        self.file_docket_path: Path | None = None
+
+        self._initialized = True
+
+    def initialize(self, root: str):
+        # Accept file or directory
+        root_path = Path(root)
+        if root_path.is_file():
+            self.root_dir = root_path.parent
+        else:
+            self.root_dir = root_path
+
+        # Repo directories
+        repo_dir = self.root_dir.parent
+        self.agent_instructions_dir = repo_dir / "agent_instructions"
+        self.data_sheets_dir = repo_dir / "data_sheets"
+        self.fw_tasks_dir = repo_dir / "fw_tasks"
+        self.generated_files_dir = repo_dir / "generated_files"
+        self.cheat_sheets_dir = repo_dir / "hw_cheat_sheets"
+        self.reqs_dir = repo_dir / "requirements"
+        self.pdf_parts_dir = repo_dir / "pdf_parts"
+        self.metadata_dir = self.root_dir / ".multifactor"
+        self.config_file_path = self.metadata_dir / "config.json"
+        self.file_docket_path = self.metadata_dir / "file_docket.json"
+
+        # Create all dirs
+        self._create_directory_structure()
+
+    def _create_directory_structure(self):
+        for directory in [
+            self.agent_instructions_dir,
+            self.data_sheets_dir,
+            self.fw_tasks_dir,
+            self.generated_files_dir,
+            self.cheat_sheets_dir,
+            self.reqs_dir,
+            self.pdf_parts_dir,
+            self.app_data_dir,
+            self.chroma_db_dir,
+            self.metadata_dir
+        ]:
+            directory.mkdir(parents=True, exist_ok=True)
+
+
+app_dirs = DirectoryManager()
+
+
+def init_directory_structure(root_dir: str):
+    app_dirs.initialize(root_dir)

mfcli/utils/file_upload.py

@@ -0,0 +1,298 @@
+"""Unified file upload abstraction for different LLM providers."""
+
+import os
+from abc import ABC, abstractmethod
+from enum import Enum
+from pathlib import Path
+from typing import Optional
+
+from google import genai
+from google.genai.types import File
+
+from mfcli.utils.config import get_config
+from mfcli.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class FileUploadProvider(str, Enum):
+    """Supported file upload providers."""
+    GEMINI = "gemini"
+    OPENAI = "openai"
+
+
+class BaseFileUploader(ABC):
+    """Base class for file upload implementations."""
+
+    @abstractmethod
+    def upload_file(self, file_path: str, display_name: Optional[str] = None) -> dict:
+        """
+        Upload a file to the provider's file storage.
+        
+        Args:
+            file_path: Path to the local file to upload
+            display_name: Optional display name for the file
+            
+        Returns:
+            Dictionary containing file metadata including URI/ID for accessing the file
+        """
+        pass
+
+    @abstractmethod
+    def delete_file(self, file_id: str) -> bool:
+        """
+        Delete a file from the provider's storage.
+        
+        Args:
+            file_id: The ID/URI of the file to delete
+            
+        Returns:
+            True if deletion was successful, False otherwise
+        """
+        pass
+
+    @abstractmethod
+    def get_file_info(self, file_id: str) -> dict:
+        """
+        Get information about an uploaded file.
+        
+        Args:
+            file_id: The ID/URI of the file
+            
+        Returns:
+            Dictionary containing file metadata
+        """
+        pass
+
+
+class GeminiFileUploader(BaseFileUploader):
+    """File uploader implementation for Google Gemini."""
+
+    def __init__(self):
+        """Initialize the Gemini file uploader."""
+        config = get_config()
+        self.client = genai.Client(api_key=config.google_api_key)
+        logger.info("Initialized Gemini file uploader")
+
+    @staticmethod
+    def _file_access_check(file_path: str):
+        file_path_obj = Path(file_path)
+
+        # Validate file exists and is readable
+        if not file_path_obj.exists():
+            raise ValueError(f"File does not exist: {file_path}")
+        if not os.access(file_path_obj, os.R_OK):
+            raise ValueError(f"File is not readable: {file_path}")
+
+    def upload(self, file_path: str) -> File:
+        """
+        Upload a file to Gemini Files API and return File object.
+
+        Args:
+            file_path: Path to the local file to upload
+
+        Returns:
+            Gemini types File object.
+
+        Raises:
+            ValueError: If file doesn't exist or is not readable
+            Exception: If upload fails
+        """
+        self._file_access_check(file_path)
+        return self.client.files.upload(file=file_path)
+
+    def upload_file(self, file_path: str, display_name: Optional[str] = None) -> dict:
+        """
+        Upload a file to Gemini Files API.
+        
+        Args:
+            file_path: Path to the local file to upload
+            display_name: Optional display name for the file
+            
+        Returns:
+            Dictionary with file metadata including 'uri', 'name', 'mime_type', 'size_bytes'
+            
+        Raises:
+            ValueError: If file doesn't exist or is not readable
+            Exception: If upload fails
+        """
+
+        self._file_access_check(file_path)
+
+        file_path_obj = Path(file_path)
+
+        # Use filename as display name if not provided
+        if display_name is None:
+            display_name = file_path_obj.name
+
+        try:
+            logger.info(f"Uploading file to Gemini: {file_path}")
+
+            # Upload the file
+            uploaded_file = self.client.files.upload(
+                file=str(file_path_obj),
+                config={'display_name': display_name}
+            )
+
+            # Extract metadata
+            result = {
+                'uri': uploaded_file.uri,
+                'name': uploaded_file.name,
+                'display_name': uploaded_file.display_name,
+                'mime_type': uploaded_file.mime_type,
+                'size_bytes': uploaded_file.size_bytes,
+                'state': uploaded_file.state.name,
+                'provider': FileUploadProvider.GEMINI.value
+            }
+
+            logger.info(f"Successfully uploaded file: {result['name']}")
+            return result
+
+        except Exception as e:
+            logger.error(f"Failed to upload file to Gemini: {e}")
+            raise Exception(f"Failed to upload file to Gemini: {str(e)}")
+
+    def delete_file(self, file_id: str) -> bool:
+        """
+        Delete a file from Gemini Files API.
+        
+        Args:
+            file_id: The name/ID of the file (e.g., 'files/abc123')
+            
+        Returns:
+            True if deletion was successful, False otherwise
+        """
+        try:
+            logger.info(f"Deleting file from Gemini: {file_id}")
+            self.client.files.delete(name=file_id)
+            logger.info(f"Successfully deleted file: {file_id}")
+            return True
+        except Exception as e:
+            logger.error(f"Failed to delete file from Gemini: {e}")
+            return False
+
+    def get_file_info(self, file_id: str) -> dict:
+        """
+        Get information about an uploaded file.
+        
+        Args:
+            file_id: The name/ID of the file (e.g., 'files/abc123')
+            
+        Returns:
+            Dictionary containing file metadata
+        """
+        try:
+            logger.info(f"Getting file info from Gemini: {file_id}")
+            file_info = self.client.files.get(name=file_id)
+
+            result = {
+                'uri': file_info.uri,
+                'name': file_info.name,
+                'display_name': file_info.display_name,
+                'mime_type': file_info.mime_type,
+                'size_bytes': file_info.size_bytes,
+                'state': file_info.state.name,
+                'provider': FileUploadProvider.GEMINI.value
+            }
+
+            return result
+
+        except Exception as e:
+            logger.error(f"Failed to get file info from Gemini: {e}")
+            raise Exception(f"Failed to get file info: {str(e)}")
+
+
+class OpenAIFileUploader(BaseFileUploader):
+    """File uploader implementation for OpenAI (placeholder for future implementation)."""
+
+    def __init__(self):
+        """Initialize the OpenAI file uploader."""
+        config = get_config()
+        # This will be implemented when OpenAI support is added
+        logger.info("OpenAI file uploader - not yet implemented")
+        raise NotImplementedError("OpenAI file upload support coming soon")
+
+    def upload_file(self, file_path: str, display_name: Optional[str] = None) -> dict:
+        """Upload a file to OpenAI."""
+        raise NotImplementedError("OpenAI file upload not yet implemented")
+
+    def delete_file(self, file_id: str) -> bool:
+        """Delete a file from OpenAI."""
+        raise NotImplementedError("OpenAI file deletion not yet implemented")
+
+    def get_file_info(self, file_id: str) -> dict:
+        """Get file info from OpenAI."""
+        raise NotImplementedError("OpenAI file info not yet implemented")
+
+
+class FileUploadManager:
+    """Manager class to handle file uploads across different providers."""
+
+    def __init__(self, provider: FileUploadProvider = FileUploadProvider.GEMINI):
+        """
+        Initialize the file upload manager.
+        
+        Args:
+            provider: The file upload provider to use (default: GEMINI)
+        """
+        self.provider = provider
+        self.uploader = self._get_uploader(provider)
+
+    def _get_uploader(self, provider: FileUploadProvider) -> BaseFileUploader:
+        """Get the appropriate uploader for the specified provider."""
+        if provider == FileUploadProvider.GEMINI:
+            return GeminiFileUploader()
+        elif provider == FileUploadProvider.OPENAI:
+            return OpenAIFileUploader()
+        else:
+            raise ValueError(f"Unsupported file upload provider: {provider}")
+
+    def upload_file(self, file_path: str, display_name: Optional[str] = None) -> dict:
+        """
+        Upload a file using the configured provider.
+        
+        Args:
+            file_path: Path to the local file to upload
+            display_name: Optional display name for the file
+            
+        Returns:
+            Dictionary with file metadata
+        """
+        return self.uploader.upload_file(file_path, display_name)
+
+    def delete_file(self, file_id: str) -> bool:
+        """
+        Delete a file using the configured provider.
+        
+        Args:
+            file_id: The ID/URI of the file to delete
+            
+        Returns:
+            True if deletion was successful, False otherwise
+        """
+        return self.uploader.delete_file(file_id)
+
+    def get_file_info(self, file_id: str) -> dict:
+        """
+        Get file information using the configured provider.
+        
+        Args:
+            file_id: The ID/URI of the file
+            
+        Returns:
+            Dictionary containing file metadata
+        """
+        return self.uploader.get_file_info(file_id)
+
+
+def get_file_upload_manager(provider: FileUploadProvider = FileUploadProvider.GEMINI) -> FileUploadManager:
+    """
+    Factory function to get a file upload manager instance.
+    
+    Args:
+        provider: The file upload provider to use (default: GEMINI)
+        
+    Returns:
+        FileUploadManager instance
+    """
+    return FileUploadManager(provider)

mfcli/utils/files.py

@@ -0,0 +1,16 @@
+import os
+import re
+from pathlib import Path
+from typing import Literal
+
+AppDataFileTypes = Literal['datasheets', 'files']
+
+
+def file_access_check(file_path: Path) -> bool:
+    if not os.path.exists(file_path) or not os.access(file_path, os.R_OK):
+        return False
+    return True
+
+
+def is_text_mime_type(mime_type: str) -> bool:
+    return bool(re.match(r"^text/.+$", mime_type))

mfcli/utils/http_requests.py

@@ -0,0 +1,54 @@
+import requests
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
+
+
+def get_retry_session(
+        retries: int = 3,
+        backoff_factor: float = 0.5,
+        status_forcelist: tuple = (500, 502, 503, 504),
+        allowed_methods: tuple = ("GET", "POST", "PUT", "DELETE", "PATCH")
+) -> requests.Session:
+    """Create a requests session with retry logic."""
+    session = requests.Session()
+    retry = Retry(
+        total=retries,
+        read=retries,
+        connect=retries,
+        backoff_factor=backoff_factor,
+        status_forcelist=status_forcelist,
+        allowed_methods=allowed_methods,
+        raise_on_status=False,
+    )
+    adapter = HTTPAdapter(max_retries=retry)
+    session.mount("http://", adapter)
+    session.mount("https://", adapter)
+    return session
+
+
+def http_request(
+        method: str,
+        url: str,
+        retries: int = 3,
+        timeout: int = 10,
+        **kwargs
+) -> requests.Response:
+    """
+    Perform an HTTP request with automatic retry logic.
+
+    :param method: HTTP method (GET, POST, PUT, DELETE, etc.)
+    :param url: URL to request
+    :param retries: Number of retry attempts
+    :param timeout: Timeout per request in seconds
+    :param kwargs: Any requests.request() parameters (headers, data, json, etc.)
+    :return: requests.Response object
+    """
+    session = get_retry_session(retries=retries)
+    try:
+        response = session.request(method.upper(), url, allow_redirects=True, timeout=timeout, **kwargs)
+        response.raise_for_status()
+        return response
+    except requests.exceptions.RequestException:
+        raise
+    finally:
+        session.close()

mfcli/utils/kb_lister.py

@@ -0,0 +1,89 @@
+from typing import Dict, List, Set
+from mfcli.client.chroma_db import get_chromadb_client_for_project_name
+from mfcli.utils.logger import get_logger
+from mfcli.utils.orm import Session
+
+logger = get_logger(__name__)
+
+
+def list_vectorized_files(project_name: str) -> Dict[str, List[str]]:
+    """
+    List all files that have been vectorized into the ChromaDB database for a project.
+    
+    Args:
+        project_name: Name of the project
+        
+    Returns:
+        Dictionary mapping purpose to list of file names
+    """
+    try:
+        with Session() as db:
+            chroma_client = get_chromadb_client_for_project_name(db, project_name)
+            
+        # Get all documents from the collection
+        collection = chroma_client._collection
+        results = collection.get()
+        
+        if not results or not results.get('metadatas'):
+            logger.info("No vectorized files found in the knowledge base")
+            return {}
+        
+        # Group files by purpose
+        files_by_purpose: Dict[str, Set[str]] = {}
+        
+        for metadata in results['metadatas']:
+            if metadata:
+                file_name = metadata.get('file_name', 'Unknown')
+                purpose = metadata.get('purpose', 'unknown')
+                
+                if purpose not in files_by_purpose:
+                    files_by_purpose[purpose] = set()
+                files_by_purpose[purpose].add(file_name)
+        
+        # Convert sets to sorted lists for consistent output
+        result = {
+            purpose: sorted(list(files)) 
+            for purpose, files in files_by_purpose.items()
+        }
+        
+        return result
+        
+    except Exception as e:
+        logger.error(f"Failed to list vectorized files for project: {project_name}")
+        logger.exception(e)
+        raise
+
+
+def print_vectorized_files(project_name: str) -> None:
+    """
+    Print all vectorized files in a formatted manner.
+    
+    Args:
+        project_name: Name of the project
+    """
+    try:
+        files_by_purpose = list_vectorized_files(project_name)
+        
+        if not files_by_purpose:
+            print(f"\nNo files have been vectorized for project '{project_name}' yet.")
+            return
+        
+        print(f"\nVectorized files in knowledge base for project '{project_name}':")
+        print("=" * 70)
+        
+        total_files = 0
+        for purpose in sorted(files_by_purpose.keys()):
+            files = files_by_purpose[purpose]
+            print(f"\n{purpose.upper()} ({len(files)} file{'s' if len(files) != 1 else ''}):")
+            print("-" * 70)
+            for file_name in files:
+                print(f"  • {file_name}")
+            total_files += len(files)
+        
+        print("\n" + "=" * 70)
+        print(f"Total: {total_files} unique file{'s' if total_files != 1 else ''} vectorized\n")
+        
+    except Exception as e:
+        logger.error(f"Failed to print vectorized files for project: {project_name}")
+        print(f"\nError: Failed to list vectorized files. Check logs for details.")
+        raise