mfcli 0.2.0__py3-none-any.whl

mfcli/pipeline/analysis/generators/functional_blocks/validator.py

@@ -0,0 +1,94 @@
+from typing import List, Dict, Tuple
+from difflib import SequenceMatcher
+
+from mfcli.models.functional_blocks import FunctionalBlockMeta
+from mfcli.models.netlist import NetlistComponent, Netlist
+from mfcli.utils.logger import get_logger
+from mfcli.utils.orm import Session
+
+logger = get_logger(__name__)
+
+
+def validate_functional_blocks_against_netlist(
+        db: Session,
+        pipeline_run_id: int,
+        functional_blocks: List[FunctionalBlockMeta]
+) -> Dict:
+    """
+    Validate functional block components against netlist.
+    Returns corrected blocks and validation report.
+    """
+
+    # Get all valid component ref_des from netlist
+    valid_components = (
+        db.query(NetlistComponent.ref_des)
+        .join(NetlistComponent.netlist)
+        .filter(Netlist.pipeline_run_id == pipeline_run_id)
+        .all()
+    )
+    valid_refs = {comp.ref_des for comp in valid_components}
+
+    report = {
+        "total_components": 0,
+        "valid_components": 0,
+        "invalid_components": [],
+        "corrected_components": [],
+        "blocks_validated": 0
+    }
+
+    for block in functional_blocks:
+        corrected_components = []
+
+        for component_ref in block.components:
+            report["total_components"] += 1
+
+            if component_ref in valid_refs:
+                # Valid component
+                report["valid_components"] += 1
+                corrected_components.append(component_ref)
+            else:
+                # Invalid - try fuzzy matching
+                best_match = find_best_match(component_ref, valid_refs)
+
+                if best_match and similarity_ratio(component_ref, best_match) > 0.8:
+                    # High confidence match - autocorrect
+                    logger.warning(f"Auto-correcting '{component_ref}' -> '{best_match}'")
+                    corrected_components.append(best_match)
+                    report["corrected_components"].append({
+                        "original": component_ref,
+                        "corrected": best_match,
+                        "block": block.name
+                    })
+                else:
+                    # No good match - log and skip
+                    logger.error(f"Invalid component '{component_ref}' in block '{block.name}'")
+                    report["invalid_components"].append({
+                        "ref": component_ref,
+                        "block": block.name,
+                        "suggestion": best_match if best_match else None
+                    })
+
+        block.components = corrected_components
+
+        report["blocks_validated"] += 1
+
+    return report
+
+
+def find_best_match(ref: str, valid_refs: set) -> str:
+    """Find nearest matching reference designator."""
+    best_match = None
+    best_ratio = 0
+
+    for valid_ref in valid_refs:
+        ratio = SequenceMatcher(None, ref.upper(), valid_ref.upper()).ratio()
+        if ratio > best_ratio:
+            best_ratio = ratio
+            best_match = valid_ref
+
+    return best_match if best_ratio > 0.6 else None
+
+
+def similarity_ratio(str1: str, str2: str) -> float:
+    """Calculate similarity between two strings."""
+    return SequenceMatcher(None, str1.upper(), str2.upper()).ratio()

mfcli/pipeline/analysis/generators/generator.py

@@ -0,0 +1,258 @@
+import csv
+import json
+from pathlib import Path
+from typing import Dict, Any, List, Type
+
+from google.genai.types import File as GeminiFile
+
+from mfcli.client.gemini import GeminiFileEntity
+from mfcli.constants.file_types import FileSubtypes
+from mfcli.models.bom import BOMSchema
+from mfcli.models.file import File
+from mfcli.models.file_docket import FileDocketEntry
+from mfcli.pipeline.analysis.generators.bom.bom import BOMGenerator
+from mfcli.pipeline.analysis.generators.debug_setup.debug_setup import DSCheatSheetGenerator
+from mfcli.pipeline.analysis.generators.functional_blocks.functional_blocks import (
+    extract_functional_blocks_from_file,
+    FBCheatSheetGenerator
+)
+from mfcli.pipeline.analysis.generators.generator_base import GeneratorBase
+from mfcli.pipeline.analysis.generators.mcu.mcu import MCUCheatSheetGenerator
+from mfcli.pipeline.analysis.generators.mcu_errata.mcu_errata import ErrataCheatSheetGenerator
+from mfcli.pipeline.analysis.generators.summary.summary import SummaryCheatSheetGenerator
+from mfcli.pipeline.run_context import PipelineRunContext
+from mfcli.utils.directory_manager import app_dirs
+from mfcli.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+FileSubtypeGeneratorMap: Dict[FileSubtypes, List[Type[GeneratorBase]]] = {
+    FileSubtypes.ERRATA: [ErrataCheatSheetGenerator],
+    FileSubtypes.MCU_DATASHEET: [MCUCheatSheetGenerator],
+    FileSubtypes.SCHEMATIC: [DSCheatSheetGenerator],
+    FileSubtypes.USER_GUIDE: [SummaryCheatSheetGenerator],
+    FileSubtypes.REFERENCE_MANUAL: [SummaryCheatSheetGenerator]
+}
+
+# Mapping of generator classes to friendly output names
+GeneratorNameMap: Dict[Any, str] = {
+    ErrataCheatSheetGenerator: "mcu_errata",
+    MCUCheatSheetGenerator: "mcu",
+    DSCheatSheetGenerator: "debug_setup",
+    FBCheatSheetGenerator: "functional_blocks",
+    SummaryCheatSheetGenerator: "summary"
+}
+
+
+class Generator:
+    def __init__(self, context: PipelineRunContext, processed_file_types: set[str] = None):
+        self._context = context
+        self._processed_file_types = processed_file_types or set()
+
+    @staticmethod
+    def _create_cheat_sheet_json_file(file_path: str, cheat_sheet_data: Dict) -> str:
+        # Ensure parent directory exists
+        Path(file_path).parent.mkdir(parents=True, exist_ok=True)
+        with open(file_path, 'w', encoding='utf-8', errors='replace') as jsonfile:
+            data = json.dumps(cheat_sheet_data, indent=2, ensure_ascii=False)
+            jsonfile.write(data)
+            return data
+
+    def _add_to_file_docket(self, file_name: str, file_path: str, vectorize: bool, sub_type: str):
+        entry = FileDocketEntry(
+            name=file_name,
+            path=str(file_path),
+            vectorize=vectorize,
+            sub_type=sub_type
+        )
+        self._context.docket.add(entry)
+
+    def _file_subtype_query(self, subtype: FileSubtypes):
+        return (
+            self._context.db.query(File)
+            .filter(File.pipeline_run_id == self._context.run.id)
+            .filter(File.sub_type == subtype)
+            .filter(File.is_datasheet == 0)
+        )
+
+    def _query_first_file_subtype(self, subtype: FileSubtypes) -> File | None:
+        return self._file_subtype_query(subtype).first()
+
+    def _query_all_file_subtype(self, subtype: FileSubtypes) -> List[File]:
+        return self._file_subtype_query(subtype).all()
+
+    @staticmethod
+    def _create_bom_csv(file_path: str, bom_entries: List[Dict]):
+        # Ensure parent directory exists
+        Path(file_path).parent.mkdir(parents=True, exist_ok=True)
+        columns = list(BOMSchema.model_fields.keys())
+        with open(file_path, "w", newline="", encoding="utf-8", errors="ignore") as f:
+            writer = csv.DictWriter(f, fieldnames=columns)
+            writer.writeheader()
+            writer.writerows(bom_entries)
+
+    def _vectorize_bom_csv(self, file_path: str):
+        with open(file_path, "r") as f:
+            self._context.vectorizer.vectorize_text_content(f.read(), "bom.csv", "BOM")
+
+    def _get_gemini_file(self, file: File) -> GeminiFile | None:
+        if not self._context.gemini_file_cache.get(file.gemini_file_id):
+            logger.warn(f"No Gemini file found for file: {file.id}")
+            return
+        return self._context.gemini_file_cache[file.gemini_file_id]
+
+    async def _generate_bom(self):
+        bom_file = self._query_first_file_subtype(FileSubtypes.BOM)
+        if bom_file:
+            return
+        logger.info("Generation BOM")
+        generator = BOMGenerator(self._context.gemini)
+        schematic_file = self._query_first_file_subtype(FileSubtypes.SCHEMATIC)
+        if not schematic_file:
+            return
+        gemini_file = self._get_gemini_file(schematic_file)
+        if not gemini_file:
+            return
+        bom_entries = await generator.generate(gemini_file)
+        file_path = app_dirs.generated_files_dir / "bom.csv"
+        self._create_bom_csv(file_path, bom_entries)
+        if self._context.config.vectorize_hw_files:
+            self._vectorize_bom_csv(file_path)
+        self._add_to_file_docket("bom.csv", file_path, self._context.config.vectorize_hw_files, "BOM")
+
+    @staticmethod
+    def _get_generator_name(generator_type: Any) -> str:
+        """Get the friendly name for a generator class."""
+        return GeneratorNameMap.get(generator_type, generator_type.__name__.lower())
+
+    def _get_cs_vectorization_setting(self, cs_type: Any) -> bool:
+        vectorize_config = self._context.config.vectorize_cheat_sheets_config
+        should_vectorize = False
+        if cs_type == ErrataCheatSheetGenerator:
+            should_vectorize = vectorize_config.vectorize_errata
+        elif cs_type == MCUCheatSheetGenerator:
+            should_vectorize = vectorize_config.vectorize_mcu
+        elif cs_type == DSCheatSheetGenerator:
+            should_vectorize = vectorize_config.vectorize_debug_setup
+        elif cs_type == FBCheatSheetGenerator:
+            should_vectorize = vectorize_config.vectorize_functional_blocks
+        return should_vectorize
+
+    def _get_gemini_files_from_pdf(self, file: File) -> List[GeminiFile]:
+        if not file.pdf_parts:
+            return [self._get_gemini_file(file)]
+        return [self._get_gemini_file(part) for part in file.pdf_parts]
+
+    async def _generate_functional_blocks(self):
+        # TODO: HANDLE CASE OF MULTIPLE SCHEMATIC/USER GUIDES/NETLISTS
+        schematic = self._query_first_file_subtype(FileSubtypes.SCHEMATIC)
+        schematic_files = self._get_gemini_files_from_pdf(schematic) if schematic else None
+        user_guide = self._query_first_file_subtype(FileSubtypes.USER_GUIDE)
+        user_guide_files = self._get_gemini_files_from_pdf(user_guide) if user_guide else None
+
+        # TODO: QUERY DIFFERENT NETLISTS
+        netlist = self._query_first_file_subtype(FileSubtypes.PROTEL_ALTIUM)
+
+        # Upload netlist to Gemini because it's not a PDF, hasn't been uploaded yet
+        netlist_file = None
+        if netlist:
+            netlist_gemini_file_meta = GeminiFileEntity(
+                path=Path(netlist.path),
+                mime_type="text/plain"
+            )
+            netlist_file = await self._context.gemini.upload(netlist_gemini_file_meta)
+
+        if schematic_files or user_guide_files or netlist_file:
+            logger.info("Generating functional blocks cheat sheet")
+
+        # Generate functional blocks from uploaded files
+        block_data = await extract_functional_blocks_from_file(
+            db=self._context.db,
+            pipeline_run=self._context.run,
+            gemini=self._context.gemini,
+            schematic_files=schematic_files,
+            user_guide_files=user_guide_files,
+            netlist_file=netlist_file
+        )
+
+        # Create cheat sheet JSON file
+        generator_name = self._get_generator_name(FBCheatSheetGenerator)
+        file_name = f"{schematic.name}_{generator_name}_cheat_sheet.json"
+        file_path = app_dirs.cheat_sheets_dir / file_name
+        text = self._create_cheat_sheet_json_file(file_path, block_data)
+
+        # Vectorize file and add to file docket
+        self._vectorize_and_add_to_docket(FBCheatSheetGenerator, file_path, file_name, text)
+
+        # Commit functional blocks to DB
+        self._context.db.commit()
+
+    def _vectorize_and_add_to_docket(self, cs_type: Any, file_path: str, file_name: str, text: str):
+        logger.debug(f"Vectorizing and adding cheat sheet to docket: {file_name}")
+        should_vectorize = self._get_cs_vectorization_setting(cs_type)
+        logger.debug(f"Should vectorize: {should_vectorize}")
+        if should_vectorize:
+            logger.debug(f"Vectorizing cheat sheet: {file_name}")
+            self._context.vectorizer.vectorize_text_content(text, file_name, "CHEAT_SHEET")
+        logger.debug(f"Adding cheat sheet to docket: {file_name}")
+        self._add_to_file_docket(file_name, file_path, should_vectorize, "CHEAT_SHEET")
+
+    async def _generate_for_subtype(self, subtype: FileSubtypes, generators: List[Type[GeneratorBase]]):
+        files = self._query_all_file_subtype(subtype)
+        for file in files:
+            gemini_files = self._get_gemini_files_from_pdf(file)
+            if not gemini_files:
+                continue
+            for generator in generators:
+                gen = generator(self._context, file, gemini_files)
+                generator_name = self._get_generator_name(generator)
+                try:
+                    logger.info(f"Generating {generator_name} cheat sheet")
+                    cheat_sheet_data = await gen.generate()
+                    if not cheat_sheet_data:
+                        continue
+                    file_name = f"{file.name}_{generator_name}_cheat_sheet.json"
+                    file_path = app_dirs.cheat_sheets_dir / file_name
+                    text = self._create_cheat_sheet_json_file(file_path, cheat_sheet_data)
+                    self._vectorize_and_add_to_docket(generator, file_path, file_name, text)
+                except Exception as e:
+                    logger.exception(e)
+                    logger.error(f"Error generating cheat sheet from {generator_name}")
+
+    async def generate_cheat_sheets(self):
+        # Only generate BOM if schematic was processed
+        if "SCHEMATIC" in self._processed_file_types:
+            logger.info("Generating BOM (schematic was processed)")
+            try:
+                await self._generate_bom()
+            except Exception as e:
+                logger.exception(e)
+                logger.error(f"Error generating BOM: {self._context.run.id}")
+        else:
+            logger.info("Skipping BOM generation (schematic not processed)")
+        
+        # Only generate functional blocks if schematic, user guide, or netlist was processed
+        netlist_types = {"PROTEL_ALTIUM", "KICAD_LEGACY_NET", "KICAD_SPICE", "PADS", "EDIF"}
+        should_generate_fb = (
+            "SCHEMATIC" in self._processed_file_types or
+            "USER_GUIDE" in self._processed_file_types or
+            any(nt in self._processed_file_types for nt in netlist_types)
+        )
+        
+        if should_generate_fb:
+            logger.info("Generating functional blocks (schematic/user guide/netlist was processed)")
+            try:
+                await self._generate_functional_blocks()
+            except Exception as e:
+                logger.exception(e)
+                logger.error(f"Error generating functional blocks: {self._context.run.id}")
+        else:
+            logger.info("Skipping functional blocks generation (no relevant files processed)")
+        
+        # Generate cheat sheets for processed file types only
+        for subtype, generators in FileSubtypeGeneratorMap.items():
+            if subtype.name in self._processed_file_types:
+                logger.info(f"Generating cheat sheets for {subtype.name} (file was processed)")
+                await self._generate_for_subtype(subtype, generators)
+            else:
+                logger.info(f"Skipping cheat sheet generation for {subtype.name} (file not processed)")

mfcli/pipeline/analysis/generators/generator_base.py

@@ -0,0 +1,18 @@
+from abc import abstractmethod
+from typing import List, Dict
+
+from google.genai.types import File as GeminiFile
+
+from mfcli.models.file import File
+from mfcli.pipeline.run_context import PipelineRunContext
+
+
+class GeneratorBase:
+    def __init__(self, context: PipelineRunContext, db_file: File, uploads: List[GeminiFile]):
+        self._context = context
+        self._file = db_file
+        self._uploads: List[GeminiFile] = uploads
+
+    @abstractmethod
+    async def generate(self) -> Dict:
+        pass

mfcli/pipeline/analysis/generators/mcu/instructions.py

@@ -0,0 +1,156 @@
+mcu_instructions_base = (
+    """
+    You are the MCU Cheat Sheet Generator agent. Your role is to analyze microcontroller
+    datasheets and user manuals to extract critical information needed for creating initial
+    firmware boot code, linker scripts, and basic "hello world" applications.
+    """
+)
+
+mcu_data_instruction = (
+    """
+    Extract the following information systematically:
+    MCU IDENTIFICATION:
+        - mcu_name: Full part number (e.g., "STM32F407VGT6", "ATMEGA328P", "MSPM0L1306")
+        - mcu_family: Family or series name
+        - architecture: CPU core architecture (ARM Cortex-M4, AVR, RISC-V, etc.)
+        - core_frequency_max: Maximum operating frequency
+    """
+)
+
+mcu_memory_map_instructions = (
+    """
+    Extract the following information systematically:
+    MEMORY MAP (CRITICAL for linker script!):
+    For each memory region, extract:
+    - name: FLASH, RAM, ROM, SRAM, etc.
+    - start_address: Starting address in hex (e.g., "0x08000000")
+    - size: Size with unit (e.g., "128KB", "0x20000")
+    - type: flash, ram, rom, peripheral, reserved
+    - description: Any important notes about this region
+    
+    Look in: Memory organization, memory map tables, address space layout sections
+    """
+)
+
+mcu_clock_config_instructions = (
+    """
+    Extract the following information systematically:
+    CLOCK CONFIGURATION:
+    - clock_source: Available clock sources (HSI, HSE, PLL, Internal RC, etc.)
+    - frequency_range: Typical or maximum frequencies
+    - configuration_steps: How to set up the system clock
+    - registers: Key clock control registers (RCC, CCR, etc.)
+    - notes: Important clock-related information
+    
+    Look in: Clock tree diagrams, RCC chapter, system clock sections
+    """
+)
+
+mcu_interrupt_vectors_instructions = (
+    """
+    Extract the following information systematically:
+    INTERRUPT VECTORS:
+    Extract the key interrupt vectors (at least the essential ones):
+    - vector_number: Position in vector table
+    - name: Interrupt name (Reset, NMI, HardFault, SysTick, etc.)
+    - address: Vector table address if specified
+    - priority: Default or configurable priority
+    - description: What triggers this interrupt
+    
+    Look in: Interrupt vector table, exception handling, NVIC sections
+    
+    Priority: Focus on:
+    - Reset vector (most important!)
+    - NMI, HardFault, SysTick (ARM Cortex)
+    - Common peripheral interrupts
+    """
+)
+
+mcu_startup_req_instructions = (
+    """
+    Extract the following information systematically:
+    STARTUP REQUIREMENTS:
+    - boot_mode: How boot mode is selected
+    - startup_code_location: Where startup code must be placed
+    - vector_table_offset: VTOR register info (ARM Cortex)
+    - stack_pointer_init: How to set up stack pointer
+    - minimum_init_sequence: Bare minimum steps to boot successfully
+    - watchdog_handling: Watchdog timer at startup
+    
+    Look in: Boot sequence, startup procedure, reset behavior sections
+    """
+)
+
+mcu_peripherals_instructions = (
+    """
+    Extract the following information systematically:
+    ESSENTIAL PERIPHERALS:
+    For GPIO, UART, or other basic peripherals:
+    - name: Peripheral name
+    - base_address: Base address in memory map
+    - clock_enable: How to enable peripheral clock
+    - initialization_steps: Basic init procedure
+    - key_registers: Important registers
+    
+    Focus on GPIO and UART for a basic hello world app
+    """
+)
+
+mcu_linker_script_instructions = (
+    """
+    Extract the following information systematically:
+    LINKER SCRIPT NOTES:
+    Provide guidance for creating a linker script:
+    - Memory region definitions (FLASH, RAM, etc.)
+    - Entry point specification
+    - Section placement (.text, .data, .bss)
+    - Stack and heap allocation
+    - Vector table placement
+    - Any special considerations
+    """
+)
+
+mcu_hello_world_instructions = (
+    """
+    Extract the following information systematically:
+    HELLO WORLD STEPS:
+    List the steps to create a minimal working program:
+    1. Set up linker script
+    2. Create startup code
+    3. Initialize system clock
+    4. Configure GPIO/UART
+    5. Write main() function
+    6. Build and flash
+    
+    Be specific about what needs to be done at each step
+    """
+)
+
+mcu_debugging_interface_instructions = (
+    """
+    Extract the following information systematically:
+    DEBUGGING/PROGRAMMING INTERFACE:
+    - debugging_interface: JTAG, SWD, ICSP, etc.
+    - programming_interface: How to program the device
+    """
+)
+
+mcu_programming_interface_instructions = (
+    """
+    Extract the following information systematically:
+    DEBUGGING/PROGRAMMING INTERFACE:
+    - debugging_interface: JTAG, SWD, ICSP, etc.
+    - programming_interface: How to program the device
+    """
+)
+
+mcu_datasheet_references = (
+    """
+    Extract the following information systematically:
+    DATASHEET REFERENCES:
+    List specific sections/pages that were most useful in this document:
+    - "Memory Map - Section 2.2, Page 45"
+    - "Clock Tree - Figure 15, Page 78"
+    etc.
+    """
+)

mfcli/pipeline/analysis/generators/mcu/mcu.py

@@ -0,0 +1,84 @@
+import asyncio
+from typing import Dict, Type, List
+
+from google.genai.types import File as GeminiFile
+from pydantic import BaseModel
+
+from mfcli.models.file import File
+from mfcli.models.mcu import (
+    MCUData,
+    MemoryMap,
+    ClockConfig,
+    InterruptVectors,
+    StartupRequirementsModel,
+    EssentialPeripherals,
+    MCULinkerScriptInfo,
+    MCUHelloWorldSteps,
+    MCUDebuggingInterface,
+    MCUProgrammingInterface,
+    MCUDatasheetReferences
+)
+from mfcli.pipeline.analysis.generators.generator_base import GeneratorBase
+from mfcli.pipeline.analysis.generators.mcu.instructions import (
+    mcu_data_instruction,
+    mcu_instructions_base,
+    mcu_memory_map_instructions,
+    mcu_clock_config_instructions,
+    mcu_interrupt_vectors_instructions,
+    mcu_startup_req_instructions,
+    mcu_peripherals_instructions,
+    mcu_linker_script_instructions,
+    mcu_hello_world_instructions,
+    mcu_debugging_interface_instructions,
+    mcu_programming_interface_instructions,
+    mcu_datasheet_references
+)
+from mfcli.pipeline.run_context import PipelineRunContext
+from mfcli.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+MCUModelInstructionsMap: Dict[Type[BaseModel], str] = {
+    MCUData: mcu_data_instruction,
+    MemoryMap: mcu_memory_map_instructions,
+    ClockConfig: mcu_clock_config_instructions,
+    InterruptVectors: mcu_interrupt_vectors_instructions,
+    StartupRequirementsModel: mcu_startup_req_instructions,
+    EssentialPeripherals: mcu_peripherals_instructions,
+    MCULinkerScriptInfo: mcu_linker_script_instructions,
+    MCUHelloWorldSteps: mcu_hello_world_instructions,
+    MCUDebuggingInterface: mcu_debugging_interface_instructions,
+    MCUProgrammingInterface: mcu_programming_interface_instructions,
+    MCUDatasheetReferences: mcu_datasheet_references
+}
+
+
+class MCUCheatSheetGenerator(GeneratorBase):
+    def __init__(self, context: PipelineRunContext, db_file: File, uploads: List[GeminiFile]):
+        super().__init__(context, db_file, uploads)
+
+    async def _extract_mcu_data(self, model: Type[BaseModel], instructions: str) -> BaseModel:
+        return await self._context.gemini.generate(
+            prompt=instructions,
+            instructions=mcu_instructions_base,
+            response_model=model,
+            files=self._uploads
+        )
+
+    @staticmethod
+    def _parse_mcu_generated_data(results: list[BaseModel]) -> Dict:
+        response_json = {}
+        for result in results:
+            response_json.update(result.model_dump())
+        return {
+            "mcu_cheat_sheet": response_json
+        }
+
+    async def generate(self) -> Dict:
+        logger.debug(f"Generating MCU data cheat sheet")
+        tasks = []
+        for model, instructions in MCUModelInstructionsMap.items():
+            tasks.append(self._extract_mcu_data(model, instructions))
+        results: list[BaseModel] = await asyncio.gather(*tasks)
+        logger.debug(f"MCU data cheat sheet generated")
+        return self._parse_mcu_generated_data(results)

mfcli/pipeline/analysis/generators/mcu_errata/__init__.py

@@ -0,0 +1 @@
+"""MCU Errata Cheat Sheet Generator for firmware-relevant silicon errata."""