mfcli 0.2.0__py3-none-any.whl

mfcli/utils/configurator.py

@@ -0,0 +1,324 @@
+"""Interactive configuration wizard for mfcli."""
+import os
+import sys
+from pathlib import Path
+from typing import Optional
+
+from mfcli.utils.directory_manager import app_dirs
+
+
+def get_env_path() -> Path:
+    """Get the path to the .env file."""
+    return app_dirs.env_file_path
+
+
+def read_existing_env() -> dict:
+    """Read existing environment variables from .env file."""
+    env_path = get_env_path()
+    env_vars = {}
+    
+    if env_path.exists():
+        with open(env_path, 'r') as f:
+            for line in f:
+                line = line.strip()
+                if line and not line.startswith('#') and '=' in line:
+                    key, value = line.split('=', 1)
+                    env_vars[key.strip()] = value.strip()
+    
+    return env_vars
+
+
+def write_env_file(env_vars: dict) -> None:
+    """Write environment variables to .env file."""
+    env_path = get_env_path()
+    env_path.parent.mkdir(parents=True, exist_ok=True)
+    
+    # Read the template
+    template_path = Path(__file__).parent.parent / '.env.example'
+    if not template_path.exists():
+        # Fallback: create basic template
+        template_content = []
+        for key in env_vars:
+            template_content.append(f"{key}={env_vars[key]}")
+        content = '\n'.join(template_content)
+    else:
+        with open(template_path, 'r') as f:
+            template_content = f.read()
+        
+        # Replace placeholder values with actual values
+        content = template_content
+        for key, value in env_vars.items():
+            # Replace the placeholder value in the template
+            content = content.replace(f"{key}=your_{key.lower()}_here", f"{key}={value}")
+            content = content.replace(f"{key}=your_{key.replace('_', ' ').lower()}_here", f"{key}={value}")
+            # Handle specific patterns
+            if key == 'google_api_key':
+                content = content.replace(f"{key}=your_google_api_key_here", f"{key}={value}")
+            elif key == 'openai_api_key':
+                content = content.replace(f"{key}=your_openai_api_key_here", f"{key}={value}")
+            elif key == 'llama_cloud_api_key':
+                content = content.replace(f"{key}=your_llamaparse_api_key_here", f"{key}={value}")
+            elif key == 'digikey_client_id':
+                content = content.replace(f"{key}=your_digikey_client_id_here", f"{key}={value}")
+            elif key == 'digikey_client_secret':
+                content = content.replace(f"{key}=your_digikey_client_secret_here", f"{key}={value}")
+    
+    with open(env_path, 'w') as f:
+        f.write(content)
+
+
+def prompt_for_value(
+    key: str,
+    description: str,
+    link: Optional[str] = None,
+    current_value: Optional[str] = None,
+    required: bool = True
+) -> Optional[str]:
+    """Prompt user for a configuration value."""
+    print(f"\n{'='*70}")
+    print(f"  {description}")
+    if link:
+        print(f"  Get your key: {link}")
+    if current_value and current_value != f"your_{key.lower()}_here":
+        print(f"  Current value: {current_value[:20]}..." if len(current_value) > 20 else f"  Current value: {current_value}")
+        prompt = f"  Enter new value (press Enter to keep current): "
+    else:
+        prompt = f"  Enter value{' (required)' if required else ' (optional)'}: "
+    
+    print(f"{'='*70}")
+    
+    value = input(prompt).strip()
+    
+    if not value:
+        if current_value and current_value != f"your_{key.lower()}_here":
+            return current_value
+        elif not required:
+            return None
+        else:
+            print("  ❌ This value is required!")
+            return prompt_for_value(key, description, link, current_value, required)
+    
+    return value
+
+
+def validate_api_key(key_name: str, api_key: str) -> bool:
+    """Validate an API key by making a test request."""
+    print(f"\n  Validating {key_name}...", end=' ')
+    sys.stdout.flush()
+    
+    try:
+        if key_name == "Google API":
+            import google.generativeai as genai
+            genai.configure(api_key=api_key)
+            # Test with a simple list models call
+            list(genai.list_models())
+            print("✅")
+            return True
+        
+        elif key_name == "OpenAI API":
+            from openai import OpenAI
+            client = OpenAI(api_key=api_key)
+            # Test with a simple models list call
+            client.models.list()
+            print("✅")
+            return True
+        
+        elif key_name == "LlamaParse API":
+            import requests
+            headers = {"Authorization": f"Bearer {api_key}"}
+            # LlamaParse doesn't have a simple test endpoint, so we'll just check format
+            if len(api_key) > 20:
+                print("✅ (format check)")
+                return True
+            else:
+                print("❌ Invalid format")
+                return False
+        
+        elif key_name == "DigiKey API":
+            # DigiKey validation would require OAuth flow, so we'll just check format
+            if len(api_key) > 10:
+                print("✅ (format check)")
+                return True
+            else:
+                print("❌ Invalid format")
+                return False
+        
+    except Exception as e:
+        print(f"❌ ({str(e)[:50]}...)")
+        return False
+    
+    return True
+
+
+def run_configuration_wizard() -> None:
+    """Run the interactive configuration wizard."""
+    print("\n" + "="*70)
+    print("  MFCLI CONFIGURATION WIZARD")
+    print("="*70)
+    print("\n  This wizard will help you configure mfcli with your API keys.")
+    print("  You can press Ctrl+C at any time to exit.\n")
+    
+    try:
+        # Read existing configuration
+        existing_env = read_existing_env()
+        new_env = existing_env.copy()
+        
+        # Google API Key
+        value = prompt_for_value(
+            "google_api_key",
+            "Google Gemini API Key",
+            "https://aistudio.google.com/app/apikey",
+            existing_env.get("google_api_key"),
+            required=True
+        )
+        if value:
+            new_env["google_api_key"] = value
+            validate_api_key("Google API", value)
+        
+        # OpenAI API Key
+        value = prompt_for_value(
+            "openai_api_key",
+            "OpenAI API Key (for embeddings)",
+            "https://platform.openai.com/api-keys",
+            existing_env.get("openai_api_key"),
+            required=True
+        )
+        if value:
+            new_env["openai_api_key"] = value
+            validate_api_key("OpenAI API", value)
+        
+        # LlamaParse API Key
+        value = prompt_for_value(
+            "llama_cloud_api_key",
+            "LlamaParse API Key (for PDF parsing)",
+            "https://cloud.llamaindex.ai/",
+            existing_env.get("llama_cloud_api_key"),
+            required=True
+        )
+        if value:
+            new_env["llama_cloud_api_key"] = value
+            validate_api_key("LlamaParse API", value)
+        
+        # DigiKey Client ID
+        value = prompt_for_value(
+            "digikey_client_id",
+            "DigiKey Client ID (for datasheet downloads)",
+            "https://developer.digikey.com/",
+            existing_env.get("digikey_client_id"),
+            required=True
+        )
+        if value:
+            new_env["digikey_client_id"] = value
+            validate_api_key("DigiKey API", value)
+        
+        # DigiKey Client Secret
+        value = prompt_for_value(
+            "digikey_client_secret",
+            "DigiKey Client Secret",
+            None,
+            existing_env.get("digikey_client_secret"),
+            required=True
+        )
+        if value:
+            new_env["digikey_client_secret"] = value
+        
+        # Embedding configuration
+        print("\n" + "="*70)
+        print("  Vector Database Configuration")
+        print("="*70)
+        print("  Using default values:")
+        print("  - Chunk size: 1000")
+        print("  - Chunk overlap: 200")
+        print("  - Embedding model: text-embedding-3-small")
+        print("  - Embedding dimensions: 1536")
+        
+        change_defaults = input("\n  Change these defaults? (y/N): ").strip().lower()
+        
+        if change_defaults == 'y':
+            value = input("  Chunk size [1000]: ").strip()
+            new_env["chunk_size"] = value if value else "1000"
+            
+            value = input("  Chunk overlap [200]: ").strip()
+            new_env["chunk_overlap"] = value if value else "200"
+            
+            value = input("  Embedding model [text-embedding-3-small]: ").strip()
+            new_env["embedding_model"] = value if value else "text-embedding-3-small"
+            
+            value = input("  Embedding dimensions [1536]: ").strip()
+            new_env["embedding_dimensions"] = value if value else "1536"
+        else:
+            new_env["chunk_size"] = existing_env.get("chunk_size", "1000")
+            new_env["chunk_overlap"] = existing_env.get("chunk_overlap", "200")
+            new_env["embedding_model"] = existing_env.get("embedding_model", "text-embedding-3-small")
+            new_env["embedding_dimensions"] = existing_env.get("embedding_dimensions", "1536")
+        
+        # Write configuration
+        write_env_file(new_env)
+        
+        env_path = get_env_path()
+        print("\n" + "="*70)
+        print("  ✅ Configuration saved successfully!")
+        print(f"  Location: {env_path}")
+        print("="*70)
+        print("\n  Next steps:")
+        print("  1. Run 'mfcli init' in your hardware project directory")
+        print("  2. Run 'mfcli run' to process your documents")
+        print("  3. (Optional) Run 'mfcli setup-mcp' to configure MCP server")
+        print("\n")
+        
+    except KeyboardInterrupt:
+        print("\n\n  ⚠️  Configuration cancelled.")
+        sys.exit(0)
+
+
+def check_configuration() -> None:
+    """Check and validate existing configuration."""
+    print("\n" + "="*70)
+    print("  CONFIGURATION CHECK")
+    print("="*70)
+    
+    env_path = get_env_path()
+    
+    if not env_path.exists():
+        print(f"\n  ❌ Configuration file not found: {env_path}")
+        print("\n  Run 'mfcli configure' to create your configuration.")
+        return
+    
+    print(f"\n  Configuration file: {env_path}")
+    
+    env_vars = read_existing_env()
+    
+    required_keys = [
+        ("google_api_key", "Google Gemini API"),
+        ("openai_api_key", "OpenAI API"),
+        ("llama_cloud_api_key", "LlamaParse API"),
+        ("digikey_client_id", "DigiKey Client ID"),
+        ("digikey_client_secret", "DigiKey Client Secret"),
+    ]
+    
+    print("\n  Checking configuration:")
+    all_valid = True
+    
+    for key, name in required_keys:
+        value = env_vars.get(key)
+        if not value or value.startswith("your_"):
+            print(f"    ❌ {name}: Not configured")
+            all_valid = False
+        else:
+            masked_value = value[:8] + "..." if len(value) > 8 else value
+            print(f"    ✅ {name}: {masked_value}")
+    
+    print("\n  Vector database configuration:")
+    print(f"    - Chunk size: {env_vars.get('chunk_size', 'Not set')}")
+    print(f"    - Chunk overlap: {env_vars.get('chunk_overlap', 'Not set')}")
+    print(f"    - Embedding model: {env_vars.get('embedding_model', 'Not set')}")
+    print(f"    - Embedding dimensions: {env_vars.get('embedding_dimensions', 'Not set')}")
+    
+    if all_valid:
+        print("\n  ✅ All required configuration values are set!")
+        print("\n  To validate API keys, run: mfcli doctor")
+    else:
+        print("\n  ⚠️  Some configuration values are missing.")
+        print("  Run 'mfcli configure' to complete your configuration.")
+    
+    print("="*70 + "\n")

mfcli/utils/data_cleaner.py

@@ -0,0 +1,82 @@
+import os.path
+import shutil
+import sys
+from pathlib import Path
+from textwrap import dedent
+from typing import List
+
+from mfcli.client.chroma_db import ChromaClient
+from mfcli.models.project import Project
+from mfcli.utils.config import get_config
+from mfcli.utils.directory_manager import app_dirs, init_directory_structure
+from mfcli.utils.logger import get_logger, setup_logging
+from mfcli.utils.orm import Session
+from mfcli.utils.query_service import QueryService
+
+logger = get_logger(__name__)
+
+warning_message = dedent(
+    """
+    
+    WARNING: This will permanently delete all mfcli data, including datasheets, cheat sheets, and project data. 
+    Should we proceed? (Y/n):
+    
+    """
+)
+
+
+class DataCleaner:
+    def __init__(self, db: Session):
+        self._db = db
+        self._query_service = QueryService(db)
+        self._config = get_config()
+
+    @staticmethod
+    def _remove_dir(dir_path: Path):
+        if not os.path.isdir(dir_path):
+            logger.warning(f"Directory does not exist: {dir_path}")
+            return
+        try:
+            shutil.rmtree(dir_path)
+        except Exception as e:
+            logger.exception(e)
+            logger.error(f"Error deleting directory: {dir_path}")
+
+    def clean(self):
+        logger.info("Cleaning mfcli data")
+        projects: List[Project] = self._query_service.query_all(Project)
+        for project in projects:
+            init_directory_structure(project.repo_dir)
+            config_dir = Path(project.repo_dir) / ".multifactor"
+            chroma_db = ChromaClient(project.index_id)
+            chroma_db.delete_collection()
+            for dir_path in [
+                config_dir,
+                app_dirs.agent_instructions_dir,
+                app_dirs.data_sheets_dir,
+                app_dirs.fw_tasks_dir,
+                app_dirs.generated_files_dir,
+                app_dirs.reqs_dir,
+                app_dirs.cheat_sheets_dir,
+                app_dirs.pdf_parts_dir
+            ]:
+                logger.debug(f"Removing directory: {dir_path}")
+                self._remove_dir(dir_path)
+            self._db.delete(project)
+            self._db.commit()
+        logger.info("All mfcli data has been cleaned")
+
+
+def run_data_cleaner():
+    with Session() as db:
+        DataCleaner(db).clean()
+
+
+def clean_app_data(user_accepted: bool = False):
+    setup_logging()
+    if not user_accepted:
+        user_input = input(warning_message)
+        if not user_input.strip() == 'Y':
+            logger.debug("User cancelled")
+            sys.exit()
+    run_data_cleaner()

mfcli/utils/datasheet_vectorizer.py

@@ -0,0 +1,281 @@
+import os
+import re
+from urllib.parse import urlparse, unquote
+
+from playwright.async_api import async_playwright, Browser
+from requests import RequestException
+from sqlmodel import select
+
+from mfcli.client.chroma_db import ChromaClient
+from mfcli.client.docling import DoclingChunker
+from mfcli.client.vector_db import DocumentVectorizer
+from mfcli.constants.file_types import PDFMimeTypes
+from mfcli.digikey.digikey import DigiKey
+from mfcli.models.bom import BOM
+from mfcli.models.datasheet import Datasheet
+from mfcli.pipeline.extractor import TextExtractor
+from mfcli.utils.directory_manager import app_dirs
+from mfcli.utils.http_requests import http_request
+from mfcli.utils.logger import get_logger
+from mfcli.utils.orm import Session
+from mfcli.utils.tools import get_mime_type_from_bytes
+
+logger = get_logger(__name__)
+
+
+class DatasheetVectorizer:
+    def __init__(self, chroma_db: ChromaClient):
+        self._extractor = TextExtractor()
+        self._vectorizer = DocumentVectorizer(chroma_db)
+        self._docling = DoclingChunker()
+
+    def _vectorize_text(self, text: str, file_name: str, purpose: str, additional_metadata: dict = None):
+        """
+        Shared method to vectorize text with metadata
+        :param text: Extracted text content
+        :param file_name: Name of the file
+        :param purpose: Purpose of the vectorization (e.g., 'datasheet', 'bom', 'errata')
+        :param additional_metadata: Optional additional metadata to include
+        """
+        metadata = {"file_name": file_name, "purpose": purpose}
+        if additional_metadata:
+            metadata.update(additional_metadata)
+        self._vectorizer.vectorize(text, metadata)
+        logger.debug(f"File vectorized: {file_name} (purpose: {purpose})")
+
+    @staticmethod
+    async def _fetch_with_playwright(browser: Browser, url: str):
+        context = await browser.new_context()
+        response = await context.request.get(url)
+        body = await response.body()
+        return body
+
+    @staticmethod
+    def _parse_ti_url(url: str) -> str:
+        """
+        Texas Instruments URLs may have goTo param which is the real URL of the PDF
+        :param url: TI URL
+        :return: URL from goTo param
+        """
+        url_query_params = urlparse(url).query
+        if not url_query_params:
+            return url
+        params = url_query_params.split('&')
+        for param in params:
+            name = param.split('=')[0]
+            value = param.split('=')[1]
+            if not name == 'gotoUrl':
+                continue
+            return unquote(value)
+        return url
+
+    @staticmethod
+    def _save_datasheet(name: str, content: bytes):
+        file_path = app_dirs.data_sheets_dir / name
+        with open(file_path, "wb") as f:
+            f.write(content)
+
+    async def _download(self, browser: Browser, url: str, purpose: str = "datasheet"):
+        logger.debug(f"Fetching datasheet: {url}")
+        try:
+            ti_url_regex = r"^https?://www.ti.com/.+$"
+            if re.match(ti_url_regex, url, re.I):
+                logger.debug(f"URL is a TI URL: {url}")
+                url = self._parse_ti_url(url)
+                logger.debug(f"Parsed TI URL: {url}")
+            url_path = urlparse(url).path
+        except ValueError as e:
+            logger.debug(f"Unable to parse datasheet URL: {url}")
+            logger.debug(e)
+            return
+        file_name = os.path.basename(url_path)
+        if not file_name.endswith(".pdf"):
+            file_name = f"{file_name}.pdf"
+        try:
+            content = http_request(method='GET', url=url).content
+            mime_type = get_mime_type_from_bytes(content, file_name)
+            if mime_type not in PDFMimeTypes:
+                logger.debug(f"Retrieved PDF is not PDF MIME type: {url}")
+                logger.debug(f"Retrying with playwright: {url}")
+                content = await self._fetch_with_playwright(browser, url)
+        except RequestException as e:
+            logger.debug(e)
+            logger.debug(f"HTTP error fetching PDF: {url}")
+            logger.debug(f"Retrying with playwright: {url}")
+            content = await self._fetch_with_playwright(browser, url)
+        except Exception as e:
+            logger.debug(f"Unhandled error fetching datasheet URL: {url}")
+            logger.debug(e)
+            return
+        mime_type = get_mime_type_from_bytes(content, file_name)
+        if mime_type not in PDFMimeTypes:
+            logger.debug(f"Could not fetch PDF even with playwright: {url}")
+            return
+        try:
+            self._save_datasheet(file_name, content)
+        except Exception as e:
+            logger.debug(e)
+            logger.debug(f"Error saving datasheet: {file_name}")
+
+    async def download(self, urls: list[str], purpose: str = "datasheet"):
+        if not urls:
+            logger.debug(f"No datasheets to vectorize, exiting")
+            return
+        logger.debug(f"Vectorizing {len(urls)} documents (purpose: {purpose})")
+        async with async_playwright() as p:
+            browser = await p.chromium.launch(headless=True)
+            try:
+                for url in urls:
+                    try:
+                        await self._download(browser, url, purpose)
+                    except Exception as e:
+                        logger.debug(e)
+                        logger.debug(f"Error processing document: {url}")
+            finally:
+                await browser.close()
+
+    def vectorize_local_file(self, file_path: str, purpose: str, additional_metadata: dict = None):
+        """
+        Vectorize a local file (e.g., generated by agents)
+        :param file_path: Path to the local file
+        :param purpose: Purpose of the vectorization (e.g., 'bom', 'errata', 'functional_blocks')
+        :param additional_metadata: Optional additional metadata to include
+        """
+        try:
+            logger.debug(f"Vectorizing local file: {file_path} (purpose: {purpose})")
+            file_name = os.path.basename(file_path)
+
+            # Check if file exists
+            if not os.path.exists(file_path):
+                logger.error(f"File does not exist: {file_path}")
+                return
+
+            # Extract text based on file type
+            with open(file_path, 'rb') as f:
+                content = f.read()
+
+            mime_type = get_mime_type_from_bytes(content, file_name)
+
+            if mime_type in PDFMimeTypes:
+                text = self._extractor.extract_pdf_bytes(content)
+            else:
+                # For non-PDF files, use the general extractor
+                text = self._extractor.extract_text_from_file_bytes(file_name, content)
+
+            logger.debug(f"Text extracted from local file: {file_path}")
+            self._vectorize_text(text, file_name, purpose, additional_metadata)
+
+        except Exception as e:
+            logger.error(f"Error vectorizing local file: {file_path}")
+            logger.exception(e)
+            raise
+
+    def vectorize_local_files(self, file_paths: list[str], purpose: str, additional_metadata: dict = None):
+        """
+        Vectorize multiple local files
+        :param file_paths: List of paths to local files
+        :param purpose: Purpose of the vectorization (e.g., 'bom', 'errata', 'functional_blocks')
+        :param additional_metadata: Optional additional metadata to include
+        """
+        if not file_paths:
+            logger.debug(f"No files to vectorize, exiting")
+            return
+
+        logger.debug(f"Vectorizing {len(file_paths)} local files (purpose: {purpose})")
+        for file_path in file_paths:
+            try:
+                self.vectorize_local_file(file_path, purpose, additional_metadata)
+            except Exception as e:
+                logger.exception(e)
+                logger.error(f"Error processing local file: {file_path}")
+        logger.debug(f"Finished vectorizing {len(file_paths)} local files")
+
+    def vectorize_text_content(self, text: str, file_name: str, purpose: str, additional_metadata: dict = None):
+        """
+        Vectorize text content directly (e.g., from agent output)
+        :param text: Text content to vectorize
+        :param file_name: Name to associate with this content
+        :param purpose: Purpose of the vectorization (e.g., 'bom', 'errata', 'functional_blocks')
+        :param additional_metadata: Optional additional metadata to include
+        """
+        try:
+            logger.debug(f"Vectorizing text content: {file_name} (purpose: {purpose})")
+            self._vectorize_text(text, file_name, purpose, additional_metadata)
+        except Exception as e:
+            logger.error(f"Error vectorizing text content: {file_name}")
+            logger.exception(e)
+            raise
+
+    def vectorize_file_buf(
+            self,
+            file_bytes: bytes,
+            file_name: str,
+            purpose: str,
+            additional_metadata: dict = None
+    ) -> None:
+        """
+        Vectorize a file from a buffer. This vectorizer uses DoclingChunker.
+        :param file_bytes: file bytes
+        :param file_name: file name
+        :param purpose: file purpose
+        :param additional_metadata: dict of metadata
+        :return: None
+        """
+        chunks = self._docling.chunk(file_name, file_bytes)
+        metadata = {"file_name": file_name, "purpose": purpose}
+        if additional_metadata:
+            metadata.update(additional_metadata)
+        self._vectorizer.vectorize_chunks(chunks, metadata)
+
+
+async def get_datasheets_for_bom_entries(db: Session, chroma_db: ChromaClient, entries: list[BOM]):
+    logger.info(f"Fetching datasheets for {len(entries)} BOM entries")
+    part_numbers = {entry.value for entry in entries}
+    logger.debug("Fetching existing datasheets for part numbers")
+
+    # Fetch existing datasheets
+    stmt = select(Datasheet).where(Datasheet.part_number.in_(part_numbers))
+    datasheets: list[Datasheet] = db.execute(stmt).scalars().all()
+    datasheet_map = {d.part_number: d.datasheet for d in datasheets}
+
+    logger.debug(f"Datasheet map: {datasheet_map}")
+    client = DigiKey()
+    new_datasheets: list[Datasheet] = []
+    datasheet_urls: list[str] = []
+    for entry in entries:
+        try:
+            logger.debug(f"Processing BOM entry: {entry.value}")
+
+            # Skip resistors, capacitors and inductors
+            ref = entry.reference
+            if ref.startswith('R') \
+            or ref.startswith('C') \
+            or ref.startswith('L') \
+            or ref.startswith('J') \
+            or ref.startswith('T') \
+            or ref.startswith('D'):
+                logger.debug(f"Skipping BOM entry {entry.value} with reference: {ref}")
+                continue
+
+            existing_datasheet = datasheet_map.get(entry.value)
+            if not existing_datasheet:
+                logger.debug(f"Datasheet does not exist for {entry.value}")
+            entry.datasheet = existing_datasheet or client.datasheet(entry.value)
+
+            # If datasheet is new create it in DB
+            if not existing_datasheet and entry.datasheet:
+                logger.debug(f"Adding new datasheet for {entry.value}: {entry.datasheet}")
+                new_datasheets.append(Datasheet(part_number=entry.value, datasheet=entry.datasheet))
+                datasheet_urls.append(entry.datasheet)
+        except Exception as e:
+            logger.error(f"Error adding datasheet for BOM entry: {entry.value}")
+            logger.exception(e)
+    if new_datasheets:
+        db.add_all(new_datasheets)
+    if datasheet_urls:
+        try:
+            await DatasheetVectorizer(chroma_db).download(datasheet_urls)
+        except Exception as e:
+            logger.error("Error vectorizing datasheets for BOM")
+            raise e
+    logger.debug("Finished adding datasheets")