mfcli 0.2.0__py3-none-any.whl

mfcli/pipeline/parsers/netlist/protel.py

@@ -0,0 +1,166 @@
+#!/usr/bin/env python3
+"""
+Protel/Altium Designer Netlist Parser
+
+Parses Protel/Altium Designer netlist files and extracts:
+- Reference designators (ref_des)
+- Part numbers/footprints
+- Pin connections (pin number + net name)
+
+Format example:
+{COMPONENT PROTEL.PCB
+  {DETAIL
+    {SUBCOMP
+      {I <footprint>.PRT <ref_des>
+        {CN
+        <pin> <net>
+        ...
+        }
+      }
+    }
+  }
+}
+"""
+
+from pathlib import Path
+from typing import Dict
+
+from mfcli.models.netlist import Component, NetlistSchema, Pin
+
+
+class ProtelParser:
+    """Parser for Protel/Altium Designer netlist files."""
+
+    def __init__(self, protel_content: str):
+        self.content = protel_content
+        self.lines = [line.strip() for line in protel_content.strip().split('\n')]
+        self.components: Dict[str, Component] = {}
+
+    def parse(self) -> NetlistSchema:
+        """Parse Protel content and return validated schema."""
+        # Validate header
+        if not self._validate_header():
+            raise ValueError("Not a valid Protel/Altium netlist file (missing {COMPONENT PROTEL.PCB header)")
+
+        # Parse components
+        self._parse_components()
+
+        # Validate and return
+        components_list = list(self.components.values())
+        return NetlistSchema(components=components_list)
+
+    def _validate_header(self) -> bool:
+        """Check if file has Protel/Altium header."""
+        return len(self.lines) > 0 and '{COMPONENT PROTEL.PCB' in self.lines[0]
+
+    def _parse_components(self):
+        """Parse components from the netlist."""
+        i = 0
+        while i < len(self.lines):
+            line = self.lines[i]
+            
+            # Look for component definition: {I <footprint>.PRT <ref_des>
+            if line.startswith('{I ') and '.PRT ' in line:
+                # Extract footprint and ref_des
+                parts = line.split()
+                if len(parts) >= 3:
+                    footprint = parts[1]  # e.g., "0603.PRT"
+                    ref_des = parts[2]    # e.g., "C1"
+                    
+                    # Remove .PRT extension from footprint
+                    if footprint.endswith('.PRT'):
+                        footprint = footprint[:-4]
+                    
+                    # Create component
+                    self.components[ref_des] = Component(
+                        ref_des=ref_des,
+                        part_number=footprint,
+                        pins=[]
+                    )
+                    
+                    # Parse pins for this component
+                    i = self._parse_pins(i + 1, ref_des)
+                    continue
+            
+            i += 1
+
+    def _parse_pins(self, start_idx: int, ref_des: str) -> int:
+        """
+        Parse pins for a component starting from the {CN block.
+        Returns the index after parsing all pins.
+        """
+        i = start_idx
+        in_cn_block = False
+        
+        while i < len(self.lines):
+            line = self.lines[i]
+            
+            # Check if we're entering the {CN block
+            if line == '{CN':
+                in_cn_block = True
+                i += 1
+                continue
+            
+            # Check if we're exiting the {CN block or component block
+            if line == '}':
+                if in_cn_block:
+                    in_cn_block = False
+                    return i + 1  # Exit after {CN block closes
+                else:
+                    return i + 1  # Exit after component block closes
+            
+            # Parse pin connections within {CN block
+            if in_cn_block and line:
+                # Format: <pin_number> <net_name>
+                # Example: "1 3V3" or "2 GND"
+                parts = line.split(None, 1)
+                if len(parts) >= 1:
+                    pin_number = parts[0]
+                    net_name = parts[1] if len(parts) > 1 else ""
+                    
+                    # Skip lines that don't start with a number (metadata)
+                    if not pin_number or not pin_number[0].isdigit():
+                        i += 1
+                        continue
+                    
+                    # Add pin to component
+                    if ref_des in self.components and net_name:
+                        pin = Pin(pin=pin_number, net=net_name)
+                        
+                        # Avoid duplicates
+                        existing_pins = self.components[ref_des].pins
+                        if not any(p.pin == pin.pin and p.net == pin.net for p in existing_pins):
+                            self.components[ref_des].pins.append(pin)
+            
+            i += 1
+        
+        return i
+
+
+def parse_protel_file(filepath: Path) -> NetlistSchema:
+    """
+    Parse a Protel/Altium Designer netlist file and return validated netlist schema.
+    
+    Args:
+        filepath: Path to Protel/Altium netlist file
+        
+    Returns:
+        NetlistSchema with components and pins
+        
+    Raises:
+        FileNotFoundError: If file doesn't exist
+        ValidationError: If parsed data doesn't match schema
+        ValueError: If file is not a valid Protel/Altium netlist
+    """
+    if not filepath.exists():
+        raise FileNotFoundError(f"Protel/Altium netlist file not found: {filepath}")
+
+    # Read file content
+    with open(filepath, 'r', encoding='utf-8', errors='replace') as f:
+        content = f.read()
+
+    # Parse
+    parser = ProtelParser(content)
+    schema = parser.parse()
+
+    return schema

mfcli/pipeline/parsers/netlist/protel_detector.py

@@ -0,0 +1,29 @@
+"""
+Protel/Altium Designer Netlist Detector
+
+Helper function to detect if a file is a Protel/Altium Designer netlist.
+"""
+
+
+def is_protel_netlist(content: str) -> bool:
+    """
+    Detect if content is from a Protel/Altium Designer netlist file.
+    
+    Args:
+        content: File content to check
+        
+    Returns:
+        True if content appears to be a Protel/Altium netlist
+    """
+    lines = content.strip().split('\n')
+    
+    if not lines:
+        return False
+    
+    # Check for Protel/Altium header in first few lines
+    for line in lines[:5]:
+        line = line.strip()
+        if '{COMPONENT PROTEL.PCB' in line:
+            return True
+    
+    return False

mfcli/pipeline/pipeline.py

@@ -0,0 +1,419 @@
+import json
+import os.path
+from pathlib import Path
+from typing import Dict, List
+
+from google.genai.types import File as GeminiFile
+
+from mfcli.models.pdf_parts import PDFPart
+from mfcli.models.project_metadata import ProjectConfig
+from mfcli.pipeline.preprocessors.user_guide import preprocess_user_guide
+from mfcli.pipeline.run_context import PipelineRunContext
+from mfcli.utils.datasheet_vectorizer import DatasheetVectorizer
+
+from mfcli.agents.tools.general import format_error_for_llm
+from mfcli.client.chroma_db import ChromaClient
+from mfcli.client.gemini import Gemini
+from mfcli.constants.directory_names import MF_PROJECT_CONFIG_DIR_NAME
+from mfcli.constants.file_types import (
+    SchemalessFileSubtypes,
+    FileTypes,
+    FileSubtypes,
+    PDFNoVectorizeFileSubtypes,
+    SummaryCheatSheetSubtypes
+)
+from mfcli.crud.file import create_file
+from mfcli.crud.pipeline_run import create_pipeline_run
+from mfcli.crud.project import get_project_by_name, read_project_config_file
+from mfcli.models.file import File
+from mfcli.models.file_docket import FileDocket, FileDocketEntry
+from mfcli.models.pipeline_run import PipelineRun
+from mfcli.models.project import Project
+from mfcli.pipeline.analysis.bom_netlist_mapper import map_netlist_to_bom_entries
+from mfcli.pipeline.analysis.generators.generator import Generator
+from mfcli.pipeline.classifier import get_file_metadata, validate_file
+from mfcli.pipeline.data_enricher import enrich_data_for_model
+from mfcli.pipeline.extractor import extract_document_text
+from mfcli.pipeline.parser import parse_schema
+from mfcli.pipeline.schema_mapper import map_schema
+from mfcli.pipeline.sub_classifier import FileSubtypeAnalyzer
+from mfcli.utils.directory_manager import app_dirs
+from mfcli.utils.logger import get_logger
+from mfcli.utils.orm import Session
+from mfcli.utils.pdf_splitter import PDFSplitter
+
+logger = get_logger(__name__)
+
+
+# TODO: IMPROVE get_file_subtype SO IT DOESN'T USE LLM
+
+
+class PipelineRunner:
+    def __init__(self, db: Session, project: Project, project_config: ProjectConfig):
+        self._db = db
+        self._project = project
+        self.folder_path = project.repo_dir
+        self.total_files = 0
+        self.successfully_processed = 0
+        self.failed_files = 0
+        self.skipped_files = 0
+        self.errors = []
+        self.pipeline_run: PipelineRun | None = None
+        self._gemini = Gemini()
+        self._gemini_file_cache: Dict[str, GeminiFile] = {}
+        self._chroma_db = ChromaClient(project.index_id)
+        self._docket = FileDocket()
+        self._vectorizer = DatasheetVectorizer(self._chroma_db)
+        self._subtype_analyzer = FileSubtypeAnalyzer(self._gemini)
+        self._config = project_config
+        self._context = PipelineRunContext(
+            db=self._db,
+            pipeline_run=self.pipeline_run,
+            gemini=self._gemini,
+            gemini_file_cache=self._gemini_file_cache,
+            docket=self._docket,
+            config=self._config,
+            vectorizer=self._vectorizer
+        )
+        # Track which file types were actually processed (not skipped) in this run
+        self._processed_file_types: set[str] = set()
+        # Load existing file docket if it exists
+        self._load_existing_docket()
+    
+    def _load_existing_docket(self):
+        """Load existing file docket from JSON file"""
+        if app_dirs.file_docket_path and app_dirs.file_docket_path.exists():
+            logger.info(f"Loading existing file docket from: {app_dirs.file_docket_path}")
+            self._docket.load_from_json(app_dirs.file_docket_path)
+        else:
+            logger.info("No existing file docket found, starting fresh")
+
+    def _add_to_file_docket(self, file: File):
+        if file.is_datasheet:
+            vectorize = self._config.vectorize_datasheets
+        else:
+            vectorize = self._config.vectorize_hw_files
+        entry = FileDocketEntry(
+            name=file.name,
+            path=file.path,
+            vectorize=vectorize,
+            sub_type=FileSubtypes(file.sub_type).name,
+            md5=file.md5,
+            is_datasheet=bool(file.is_datasheet)
+        )
+        self._docket.add(entry)
+
+    def _save_file_docket(self):
+        json_data = json.dumps(self._docket.get_entries(), indent=2)
+        with open(app_dirs.file_docket_path, "w") as f:
+            f.write(json_data)
+
+    async def _gemini_files_upload(self, files: List[File | PDFPart]) -> List[GeminiFile]:
+        gemini_files = []
+        for file in files:
+            logger.debug(f"Checking for {type(file)} Gemini file")
+            if file.gemini_file_id in self._gemini_file_cache:
+                gemini_files.append(self._gemini_file_cache[file.gemini_file_id])
+                continue
+            logger.debug(f"Uploading {type(file)} to Gemini API")
+            gemini_file = await self._gemini.upload(file.path)
+            self._gemini_file_cache[gemini_file.name] = gemini_file
+            file.gemini_file_id = gemini_file.name
+            gemini_files.append(gemini_file)
+        return gemini_files
+
+    async def _preprocess_pdf(self, file: File, file_path: str, content: bytes):
+        logger.debug(f"Uploading file to Gemini: {file_path}")
+        splitter = PDFSplitter(file.name, content)
+        logger.debug("Splitting PDF head")
+        pdf_head_path = splitter.split_pdf_head()
+        gemini_pdf_head_file = await self._gemini.upload(pdf_head_path)
+        logger.debug(f"Analyzing PDF subtype: {file_path}")
+        await self._subtype_analyzer.analyze_pdf(file, gemini_pdf_head_file)
+
+        # Pre-process user guide files which are too big to upload to Gemini normally
+        # Extract the table of contents and split PDF into relevant content sections
+        # These sections will be used to generate summaries in analysis phase of pipeline
+        if file.sub_type in SummaryCheatSheetSubtypes:
+            await preprocess_user_guide(
+                context=self._context,
+                file=file,
+                pdf_head=gemini_pdf_head_file,
+                content=content,
+                splitter=splitter
+            )
+        else:
+            gemini_files = await self._gemini_files_upload([file])
+            logger.debug(f"Gemini files: {gemini_files}")
+
+        if self._config.vectorize_hw_files:
+            if file.sub_type in PDFNoVectorizeFileSubtypes:
+                logger.debug(f"PDF subtype does not require vectorization, skipping")
+            else:
+                logger.info(f"Chunking and vectorizing PDF: {file_path}")
+
+                # Use Docling to chunk PDF file and then vectorize it
+                self._vectorizer.vectorize_file_buf(
+                    file_name=file.name,
+                    file_bytes=content,
+                    purpose=FileSubtypes(file.sub_type).name
+                )
+        else:
+            logger.debug(f"vectorize_hw_files is set to False, skipping")
+
+    async def _preprocess_other_file_types(self, file: File, file_path: str, content: bytes):
+        logger.debug(f"File is not a PDF: {file_path}")
+        text_content = extract_document_text(file, content)
+        logger.debug(f"Analyzing file subtype: {file_path}")
+
+        # Analyze subtype
+        await self._subtype_analyzer.analyze_file(file, text_content)
+
+        if self._config.vectorize_hw_files:
+            logger.info(f"Vectorizing file: {file_path}")
+
+            # Vectorize
+            self._vectorizer.vectorize_text_content(
+                text=text_content,
+                file_name=file.name,
+                purpose=FileSubtypes(file.sub_type).name,
+                additional_metadata={"is_datasheet": file.is_datasheet}
+            )
+
+        # Ignore file subtypes that have no schema to parse like schematic files
+        if file.sub_type in SchemalessFileSubtypes:
+            logger.debug(f"File subtype is in ignore list, not parsing")
+        else:
+            logger.debug(f"File subtype is not in ignore list, parsing")
+
+            # Map schema
+            logger.debug(f"Mapping schema: {file_path}")
+            schema_mapping = await map_schema(self._gemini, file.sub_type, text_content)
+
+            # Parse schema from file
+            instances = parse_schema(self._db, file, schema_mapping)
+
+            # Enrich data
+            await enrich_data_for_model(self._db, self._chroma_db, file.sub_type, instances)
+
+    async def _preprocess_file(self, file_path: str, is_datasheet: bool) -> File:
+        logger.debug(f"Initializing Gemini client for pipeline: {self.pipeline_run.id}")
+
+        # Retrieve metadata
+        logger.debug(f"Retrieving file metadata: {file_path}")
+        metadata, content = get_file_metadata(file_path, is_datasheet)
+        logger.debug(f"Metadata retrieved: {file_path}")
+
+        # Validate file
+        validate_file(metadata)
+        logger.debug(f"File type validated: {file_path}")
+
+        # Create file
+        file = create_file(self._db, self.pipeline_run.id, metadata)
+
+        # PDF file pre-processing
+        if file.type == FileTypes.PDF:
+            await self._preprocess_pdf(file, file_path, content)
+
+        # Other file types pre-processing
+        else:
+            await self._preprocess_other_file_types(file, file_path, content)
+
+        self._db.commit()
+        logger.info(f"Pre-processing complete: {file_path}")
+        return file
+
+    async def _run_analysis(self):
+        logger.debug(f"Starting analysis for pipeline: {self.pipeline_run.id}")
+        
+        # Only run netlist-to-BOM mapping if BOM or netlist files were processed
+        netlist_types = {"PROTEL_ALTIUM", "KICAD_LEGACY_NET", "KICAD_SPICE", "PADS", "EDIF"}
+        should_map_netlist = (
+            "BOM" in self._processed_file_types or 
+            any(nt in self._processed_file_types for nt in netlist_types)
+        )
+        
+        if should_map_netlist:
+            logger.info("Running netlist-to-BOM mapping (BOM or netlist files were processed)")
+            map_netlist_to_bom_entries(self._db, self.pipeline_run.id)
+        else:
+            logger.info("Skipping netlist-to-BOM mapping (no BOM or netlist files processed)")
+        
+        logger.debug(f"Finished analysis for pipeline: {self.pipeline_run.id}")
+        
+        # Pass processed file types to generator for conditional generation
+        generator = Generator(self._context, self._processed_file_types)
+        await generator.generate_cheat_sheets()
+
+    def _check_file_in_docket(self, file_path: str, file_md5: str) -> tuple[bool, bool]:
+        """
+        Check if file exists in docket and if MD5 matches.
+        Returns: (should_process, should_remove_old)
+        """
+        # Check if file with same path exists in docket
+        existing_entry = self._docket.get_by_path(file_path)
+        
+        if not existing_entry:
+            # File not in docket, process it
+            return True, False
+        
+        # File exists in docket, check MD5
+        if existing_entry.md5 == file_md5:
+            # MD5 matches, skip processing
+            logger.info(f"File already processed with matching MD5, skipping: {file_path}")
+            return False, False
+        
+        # MD5 doesn't match, prompt user
+        logger.warning(f"File exists in docket but MD5 has changed: {file_path}")
+        logger.warning(f"  Old MD5: {existing_entry.md5}")
+        logger.warning(f"  New MD5: {file_md5}")
+        
+        print(f"\n{'='*70}")
+        print(f"File has been modified: {Path(file_path).name}")
+        print(f"Path: {file_path}")
+        print(f"Old MD5: {existing_entry.md5}")
+        print(f"New MD5: {file_md5}")
+        print(f"{'='*70}")
+        response = input("Do you want to delete the old file data and process the new version? (yes/no): ").strip().lower()
+        
+        if response in ['yes', 'y']:
+            logger.info(f"User confirmed deletion and reprocessing of: {file_path}")
+            return True, True
+        else:
+            logger.info(f"User declined reprocessing, skipping: {file_path}")
+            return False, False
+
+    def _remove_file_from_kb(self, entry: FileDocketEntry):
+        """Remove file from knowledge base (ChromaDB)"""
+        try:
+            collection = self._chroma_db._collection
+            results = collection.get()
+            
+            if not results or not results.get('metadatas'):
+                logger.warning(f"No data found in knowledge base to remove for: {entry.name}")
+                return
+            
+            # Find matching chunks for this file
+            matching_ids = []
+            for idx, metadata in enumerate(results['metadatas']):
+                if metadata and metadata.get('file_name') == entry.name:
+                    matching_ids.append(results['ids'][idx])
+            
+            if matching_ids:
+                collection.delete(ids=matching_ids)
+                logger.info(f"Removed {len(matching_ids)} chunks from knowledge base for: {entry.name}")
+            else:
+                logger.info(f"No chunks found in knowledge base for: {entry.name}")
+                
+            # Remove from docket
+            self._docket.remove(entry)
+            logger.info(f"Removed file from docket: {entry.name}")
+            
+        except Exception as e:
+            logger.error(f"Error removing file from knowledge base: {entry.name}")
+            logger.exception(e)
+
+    async def _preprocess_folder(self, folder_path: str | Path, is_datasheet: bool):
+        ignore_dirs = [MF_PROJECT_CONFIG_DIR_NAME]
+        for dir_path, dir_names, file_names in os.walk(folder_path):
+            dir_names[:] = [d for d in dir_names if d not in ignore_dirs]
+            for file_name in file_names:
+                self.total_files += 1
+                file_path = os.path.join(dir_path, file_name)
+                
+                try:
+                    # Get file metadata to check MD5
+                    logger.debug(f"Checking file: {file_path}")
+                    metadata, _ = get_file_metadata(file_path, is_datasheet)
+                    
+                    # Check if file should be processed
+                    should_process, should_remove_old = self._check_file_in_docket(file_path, metadata.md5)
+                    
+                    if not should_process:
+                        # Skip this file
+                        self.skipped_files += 1
+                        logger.info(f"Skipping file: {file_path}")
+                        continue
+                    
+                    # If we need to remove old version first
+                    if should_remove_old:
+                        existing_entry = self._docket.get_by_path(file_path)
+                        if existing_entry:
+                            logger.info(f"Removing old version from knowledge base: {file_path}")
+                            self._remove_file_from_kb(existing_entry)
+                    
+                    # Process the file
+                    logger.info(f"Pre-processing file: {file_path}")
+                    file = await self._preprocess_file(file_path, is_datasheet)
+                    self.successfully_processed += 1
+                    self._add_to_file_docket(file)
+                    
+                    # Track the file subtype as processed
+                    if file.sub_type:
+                        self._processed_file_types.add(FileSubtypes(file.sub_type).name)
+                    
+                except Exception as e:
+                    self.failed_files += 1
+                    self.errors.append({"file_path": file_path, "error": str(e)})
+                    logger.exception(e)
+                    logger.error(f"Error processing file: {file_path}")
+
+    async def run(self):
+        try:
+            logger.info(f"Starting pipeline for directory: {self.folder_path}")
+            self.pipeline_run = create_pipeline_run(self._db, self._project)
+            self._context.run = self.pipeline_run
+            await self._preprocess_folder(self.folder_path, False)
+            logger.info(f"Finished pre-processing folder: {self.folder_path}")
+
+            # Run pre-processing on datasheets which were just downloaded
+            logger.info(f"Starting pre-processing of datasheets: {app_dirs.data_sheets_dir}")
+            await self._preprocess_folder(app_dirs.data_sheets_dir, True)
+            logger.info(f"Finished pre-processing of datasheets: {app_dirs.data_sheets_dir}")
+
+            logger.info(f"Preprocessing finished: {self.folder_path}")
+            logger.info(f"Running analysis step: {self.folder_path}")
+            await self._run_analysis()
+            self._db.commit()
+            self._save_file_docket()
+            report = json.dumps({
+                "total_files": self.total_files,
+                "successfully_processed": self.successfully_processed,
+                "skipped_files": self.skipped_files,
+                "failed_files": self.failed_files,
+                "errors": self.errors
+            })
+            logger.info(f"Finished pipeline")
+            logger.info(f"Report: {report}")
+            
+            # Print summary to console
+            print(f"\n{'='*70}")
+            print(f"Pipeline Execution Summary")
+            print(f"{'='*70}")
+            print(f"Total files found: {self.total_files}")
+            print(f"Successfully processed: {self.successfully_processed}")
+            print(f"Skipped (already processed): {self.skipped_files}")
+            print(f"Failed: {self.failed_files}")
+            print(f"{'='*70}\n")
+        except Exception as e:
+            logger.exception(e)
+            logger.error(f"Error in pipeline: {e}")
+            return format_error_for_llm(e)
+
+
+async def run_pipeline_with_config(project_config: ProjectConfig):
+    with Session() as db:
+        project = get_project_by_name(db, project_config.name)
+        return await PipelineRunner(db, project, project_config).run()
+
+
+async def run_pipeline(project_name: str) -> str:
+    """
+    The controller agent will call this tool to start the pipeline processing for all the files in a directory.
+    :param project_name: The name of the project
+    :return: Status of the pipeline run
+    """
+    with Session() as db:
+        project = get_project_by_name(db, project_name)
+        project_config = read_project_config_file()
+        return await PipelineRunner(db, project, project_config).run()