mfcli 0.2.0__py3-none-any.whl

mfcli/models/netlist.py

@@ -0,0 +1,59 @@
+from datetime import datetime
+from typing import List, Optional
+
+from pydantic import BaseModel
+from sqlalchemy import Column, TIMESTAMP, func
+from sqlmodel import Field, Relationship, SQLModel
+
+from mfcli.models.bom import BOM
+from mfcli.models.pipeline_run import PipelineRun
+
+
+class Pin(BaseModel):
+    """Represents a single pin connection."""
+    pin: str = Field(..., description="Pin number or name")
+    net: str = Field(..., description="Net name this pin connects to")
+
+
+class Component(BaseModel):
+    """Represents a component with its pins and connections."""
+    ref_des: str = Field(..., description="Reference designator (C1, R5, U2, etc.)")
+    part_number: str = Field(..., description="Part number or footprint")
+    pins: List[Pin] = Field(default_factory=list, description="List of pin connections")
+
+
+class NetlistSchema(BaseModel):
+    """Top-level schema for parsed netlist."""
+    components: List[Component] = Field(default_factory=list, description="List of components")
+
+
+class Netlist(SQLModel, table=True):
+    __tablename__ = "netlists"
+    id: Optional[int] = Field(default=None, primary_key=True)
+    pipeline_run_id: int = Field(foreign_key='pipeline_runs.id', index=True, nullable=False, ondelete="CASCADE")
+    pipeline_run: Optional["PipelineRun"] = Relationship()
+    components: List["NetlistComponent"] = Relationship(back_populates='netlist', cascade_delete=True)
+    created_at: datetime = Field(
+        sa_column=Column(TIMESTAMP(timezone=True), server_default=func.now(), nullable=False)
+    )
+
+
+class NetlistComponent(SQLModel, table=True):
+    __tablename__ = "netlist_components"
+    id: Optional[int] = Field(default=None, primary_key=True)
+    netlist_id: int = Field(foreign_key='netlists.id', index=True, nullable=False, ondelete="CASCADE")
+    netlist: Optional["Netlist"] = Relationship(back_populates='components')
+    pins: List["NetlistPin"] = Relationship(back_populates='netlist_component', cascade_delete=True)
+    ref_des: str = Field(..., max_length=100, nullable=False)
+    part_number: str = Field(max_length=255, nullable=False, index=True)
+    bom_entry_id: Optional[int] = Field(foreign_key='bom.id', index=True, ondelete='CASCADE')
+    bom_entry: Optional["BOM"] = Relationship()
+
+
+class NetlistPin(SQLModel, table=True):
+    __tablename__ = "netlist_pins"
+    id: Optional[int] = Field(default=None, primary_key=True)
+    netlist_component_id: int = Field(foreign_key='netlist_components.id', index=True, nullable=False, ondelete="CASCADE")
+    netlist_component: NetlistComponent = Relationship()
+    pin: str = Field(..., max_length=100, nullable=False)
+    net: str = Field(..., max_length=100, nullable=False)

mfcli/models/pdf_parts.py

@@ -0,0 +1,25 @@
+from typing import Optional
+
+from sqlalchemy import Column, Integer, ForeignKey
+from sqlmodel import SQLModel, Field, Relationship
+
+
+class PDFPart(SQLModel, table=True):
+    __tablename__ = "pdf_parts"
+
+    id: Optional[int] = Field(default=None, primary_key=True)
+    file_id: int = Field(
+        sa_column=Column(
+            Integer,
+            ForeignKey("files.id", ondelete="CASCADE"),
+            nullable=False,
+            index=True
+        )
+    )
+    file: Optional["File"] = Relationship(back_populates='pdf_parts')
+    path: str = Field(max_length=600, nullable=False)
+    gemini_file_id: str = Field(max_length=40, nullable=True)
+    start_page: int = Field(nullable=True)
+    end_page: int = Field(nullable=True)
+    title: str = Field(nullable=True)
+    section_no: int = Field(nullable=True)

mfcli/models/pipeline_run.py

@@ -0,0 +1,34 @@
+from datetime import datetime
+from typing import Optional
+
+from sqlalchemy import Integer, func, Text, Column, DateTime, ForeignKey
+from sqlmodel import SQLModel, Field, Relationship
+
+from mfcli.constants.pipeline_run_status import PIPELINE_STATUS_IN_PROGRESS
+from mfcli.models.project import Project
+
+
+class PipelineRun(SQLModel, table=True):
+    __tablename__ = "pipeline_runs"
+
+    id: Optional[int] = Field(default=None, primary_key=True)
+    project_id: int = Field(
+        sa_column=Column(
+            Integer,
+            ForeignKey("projects.id", ondelete="CASCADE"),
+            nullable=True,
+            index=True
+        )
+    )
+    project: Optional["Project"] = Relationship(back_populates='runs')
+    status: int = Field(
+        sa_column=Column(Integer, nullable=False, server_default=str(PIPELINE_STATUS_IN_PROGRESS))
+    )
+    errors: Optional[str] = Field(default=None, sa_column=Column(Text, nullable=True))
+
+    created_at: datetime = Field(
+        sa_column=Column(DateTime(timezone=True), server_default=func.now(), nullable=False)
+    )
+    updated_at: datetime = Field(
+        sa_column=Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now(), nullable=False)
+    )

mfcli/models/project.py

@@ -0,0 +1,27 @@
+from datetime import datetime
+from typing import Optional, List
+
+from sqlalchemy import Column, DateTime, func
+from sqlmodel import SQLModel, Field, Relationship
+
+project_name_regex = r"^[A-Za-z0-9_-]+$"
+
+
+class Project(SQLModel, table=True):
+    __tablename__ = "projects"
+
+    id: Optional[int] = Field(default=None, primary_key=True)
+    runs: List["PipelineRun"] = Relationship(back_populates='project', cascade_delete=True)
+    name: str = Field(
+        nullable=False,
+        unique=True,
+        index=True,
+        min_length=3,
+        max_length=45,
+        regex=project_name_regex
+    )
+    index_id: str = Field(index=True, unique=True)
+    repo_dir: str = Field(index=True, unique=True)
+    created_at: datetime = Field(
+        sa_column=Column(DateTime(timezone=True), server_default=func.now(), nullable=False)
+    )

mfcli/models/project_metadata.py

@@ -0,0 +1,15 @@
+from pydantic import BaseModel, Field
+
+
+class VectorizeCheatSheetsConfig(BaseModel):
+    vectorize_errata: bool = Field(default=False)
+    vectorize_mcu: bool = Field(default=False)
+    vectorize_debug_setup: bool = Field(default=False)
+    vectorize_functional_blocks: bool = Field(default=False)
+
+
+class ProjectConfig(BaseModel):
+    name: str
+    vectorize_hw_files: bool = Field(default=True)
+    vectorize_datasheets: bool = Field(default=True)
+    vectorize_cheat_sheets_config: VectorizeCheatSheetsConfig = Field(default_factory=VectorizeCheatSheetsConfig)

mfcli/pipeline/analysis/bom_netlist_mapper.py

@@ -0,0 +1,28 @@
+from mfcli.models.bom import BOM
+from mfcli.models.file import File
+from mfcli.models.netlist import NetlistComponent, Netlist
+from mfcli.utils.logger import get_logger
+from mfcli.utils.orm import Session
+
+logger = get_logger(__name__)
+
+
+def map_netlist_to_bom_entries(db: Session, pipeline_run_id: int):
+    logger.debug(f"Mapping netlist to BOM entries for pipeline: {pipeline_run_id}")
+    components: list[NetlistComponent] = (
+        db.query(NetlistComponent)
+        .join(NetlistComponent.netlist)
+        .filter(Netlist.pipeline_run_id == pipeline_run_id)
+        .all()
+    )
+    results = (
+        db.query(BOM.id, BOM.value)
+        .join(BOM.file)
+        .filter(File.pipeline_run_id == pipeline_run_id)
+        .all()
+    )
+    bom_value_id_map = {result[1]: result[0] for result in results}
+    for component in components:
+        if bom_value_id_map.get(component.part_number):
+            component.bom_entry_id = bom_value_id_map[component.part_number]
+    logger.debug(f"All netlist entries mapped: {pipeline_run_id}")

mfcli/pipeline/analysis/generators/bom/bom.py

@@ -0,0 +1,74 @@
+import asyncio
+from asyncio import Semaphore
+from typing import Dict, Type, List
+
+from google.genai.types import File
+from pydantic import BaseModel, Field
+
+from mfcli.client.gemini import Gemini
+from mfcli.models.bom import BOMSchema
+from mfcli.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+system_instructions = (
+    """
+    You are the BOM Generator agent. Your role is to extract components from PDF schematics
+    and generate a Bill of Materials (BOM) in CSV format.
+    """
+)
+
+identify_bom_entries_instructions = (
+    """
+    You will return a list of names for Bill of Materials (BOM) entries from this schematic file.
+    """
+)
+
+generate_bom_entry_instructions = (
+    """
+    Analyze the schematic and extract BOM entry data for component: {}:
+    - Reference designators (R1, C1, U1, etc.)
+    - Component values (10kΩ, 100nF, etc.)
+    - Part numbers and manufacturers (if visible)
+    - Footprints/packages (if visible)
+    - Any other available information
+    """
+)
+
+
+class BOMEntryNames(BaseModel):
+    names: List[str] = Field(default_factory=list, description="BOM entry names")
+
+
+class BOMGenerator:
+    def __init__(self, gemini: Gemini):
+        self._gemini = gemini
+        self._sem = Semaphore(5)
+
+    async def _generate(self, file: File, model: Type[BaseModel], instructions: str) -> BaseModel:
+        async with self._sem:
+            return await self._gemini.generate(
+                prompt=instructions,
+                instructions=system_instructions,
+                response_model=model,
+                files=[file]
+            )
+
+    async def _get_bom_entry_names(self, file: File) -> List[str]:
+        response = await self._generate(file, BOMEntryNames, identify_bom_entries_instructions)
+        return response.names
+
+    async def generate(self, file: File) -> List[Dict]:
+        bom_entry_names = await self._get_bom_entry_names(file)
+        tasks = []
+        for name in bom_entry_names:
+            bom_entry_instructions = generate_bom_entry_instructions.format(name)
+            tasks.append(self._generate(file, BOMSchema, bom_entry_instructions))
+        results: List[BOMSchema | Exception] = await asyncio.gather(*tasks, return_exceptions=True)
+        response_data = []
+        for result in results:
+            if isinstance(result, Exception):
+                logger.warn(f"There was an error generating BOM entry data: {result}")
+                continue
+            response_data.append(result.model_dump())
+        return response_data

mfcli/pipeline/analysis/generators/debug_setup/debug_setup.py

@@ -0,0 +1,71 @@
+import asyncio
+from asyncio import Semaphore
+from typing import Dict, Type, List
+
+from google.genai.types import File as GeminiFile
+from pydantic import BaseModel
+
+from mfcli.models.debug_setup import (
+    DSTargetMCU,
+    DebugInterface,
+    DebugTool,
+    DriverInfo,
+    VSCodeLaunchConfig,
+    DSAdditionalTools,
+    DSTroubleshootingGuide,
+    DSQuickStartGuide
+)
+from mfcli.models.file import File
+from mfcli.pipeline.analysis.generators.debug_setup.instructions import (
+    ds_mcu_instructions,
+    ds_debug_int_instructions,
+    ds_recommended_tool_instructions,
+    ds_drivers_instructions,
+    ds_vscode_launch_instructions,
+    ds_add_tools_instructions,
+    ds_troubleshooting_instructions,
+    ds_quick_start_guide_instructions,
+    ds_system_instructions
+)
+from mfcli.pipeline.analysis.generators.generator_base import GeneratorBase
+from mfcli.pipeline.run_context import PipelineRunContext
+from mfcli.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+DSModelInstructionsMap = {
+    DSTargetMCU: ds_mcu_instructions,
+    DebugInterface: ds_debug_int_instructions,
+    DebugTool: ds_recommended_tool_instructions,
+    DriverInfo: ds_drivers_instructions,
+    VSCodeLaunchConfig: ds_vscode_launch_instructions,
+    DSAdditionalTools: ds_add_tools_instructions,
+    DSTroubleshootingGuide: ds_troubleshooting_instructions,
+    DSQuickStartGuide: ds_quick_start_guide_instructions
+}
+
+
+class DSCheatSheetGenerator(GeneratorBase):
+    def __init__(self, context: PipelineRunContext, db_file: File, uploads: List[GeminiFile]):
+        super().__init__(context, db_file, uploads)
+        self._sem = Semaphore(5)
+
+    async def _extract_ds_data(self, model: Type[BaseModel], instructions: str) -> BaseModel:
+        return await self._context.gemini.generate(
+            prompt=instructions,
+            instructions=ds_system_instructions,
+            response_model=model,
+            files=self._uploads
+        )
+
+    async def generate(self) -> Dict:
+        tasks = []
+        for model, instructions in DSModelInstructionsMap.items():
+            tasks.append(self._extract_ds_data(model, instructions))
+        results: list[BaseModel] = await asyncio.gather(*tasks)
+        response_dict = {}
+        for result in results:
+            response_dict.update(result.model_dump())
+        return {
+            "debug_setup": response_dict
+        }

mfcli/pipeline/analysis/generators/debug_setup/instructions.py

@@ -0,0 +1,150 @@
+ds_system_instructions = (
+    """
+    You are the Debug Setup Cheat Sheet Generator agent. Your role is to analyze schematics
+    to extract debug interface information and provide comprehensive setup guidance including:
+    - Recommended JTAG/SWD tools based on the MCU and interface
+    - Driver installation for Windows and Linux
+    - VS Code launch.json configuration
+    - GDB server setup
+    - Quick start guide
+    
+    
+    Analyze the schematic to identify debug interface:
+    Look for:
+    - Debug connector (J-TAG, SWD, ICSP headers)
+    - Pin labels (SWDIO, SWCLK, TMS, TCK, TDI, TDO, etc.)
+    - Target MCU identification
+    - Voltage levels
+    - Connector type and pinout
+    
+    Determine recommended debug tool based on MCU:
+    
+    Common mappings:
+    - STM32 → ST-Link V3 (official), J-Link (professional)
+    - NXP LPC/Kinetis → LPC-Link2, J-Link
+    - Nordic nRF → J-Link, nRF DK
+    - TI MSP430/MSPM0 → MSP-FET, XDS110
+    - Microchip PIC/AVR → PICkit, ICD, MPLAB Snap
+    - ARM Cortex-M (generic) → CMSIS-DAP, J-Link
+    - ESP32 → ESP-Prog, J-Link
+    - RISC-V → J-Link, OpenOCD compatible probes
+    """
+)
+
+ds_mcu_instructions = (
+    """
+    Extract information systematically:
+    
+    TARGET MCU:
+    - Identify the MCU from schematic
+    - Note the architecture (ARM Cortex-M4, AVR, etc.)
+    """
+)
+
+ds_debug_int_instructions = (
+    """
+    Extract information systematically:
+
+    DEBUG INTERFACE:
+    - interface_type: JTAG, SWD, ICSP, PDI, UPDI, etc.
+    - pin_count: Number of pins on debug connector
+    - pin_mapping: Map pin numbers to signals
+    Example: {"1": "VCC", "2": "SWDIO", "3": "GND", "4": "SWCLK"}
+    - connector_type: Physical connector (10-pin Cortex, 20-pin JTAG, etc.)
+    - voltage_level: Target voltage (typically 3.3V or 5V)
+    """
+)
+
+ds_recommended_tool_instructions = (
+    """
+    Extract information systematically:
+
+    RECOMMENDED TOOL:
+    - name: Primary tool recommendation (e.g., "J-Link", "ST-Link V3")
+    - model: Specific model if applicable
+    - purchase_links: Where to buy (Digikey, Mouser, official sites)
+    - compatibility_notes: Why this tool is recommended
+    - alternative_tools: Other compatible options
+    """
+)
+
+ds_drivers_instructions = (
+    """
+    Extract information systematically:
+
+    1. DRIVERS (Windows):
+    - download_url: Where to download drivers
+    - installation_steps: Step-by-step installation for Windows
+    Example:
+    1. Download from [URL]
+    2. Run installer
+    3. Connect debugger
+    4. Verify in Device Manager
+    
+    2. DRIVERS (Linux):
+    - package_name: Package to install (if available)
+    - installation_steps: Commands to install
+    Example for J-Link:
+    1. Download from segger.com
+    2. sudo dpkg -i JLink_*.deb
+    3. Add udev rules
+    4. sudo usermod -a -G plugdev $USER
+    
+    3. GDB SERVER:
+    - server_name: GDB server to use (JLinkGDBServer, OpenOCD, pyOCD, etc.)
+    - command_line: How to start server
+    Example: "JLinkGDBServer -device STM32F407VG -if SWD -speed 4000"
+    - config_file: Configuration if needed (OpenOCD .cfg)
+    - port: GDB port (default 3333)
+    """
+)
+
+ds_vscode_launch_instructions = (
+    """
+    Extract information systematically:
+
+    VS CODE LAUNCH.JSON:
+    - configuration_name: Name for the config (e.g., "Debug with J-Link")
+    - configuration_type: cortex-debug, cppdbg, etc.
+    - launch_json: Complete configuration object for .vscode/launch.json
+    - extensions_required: VS Code extensions needed
+    Common: "marus25.cortex-debug", "ms-vscode.cpptools"
+    - setup_notes: Any additional VS Code setup
+    """
+)
+
+ds_add_tools_instructions = (
+    """
+    Extract information systematically:
+
+    ADDITIONAL TOOLS:
+    - List helpful tools: OpenOCD, pyOCD, gdb-multiarch, etc.
+    """
+)
+
+ds_troubleshooting_instructions = (
+    """
+    Extract information systematically:
+    
+    TROUBLESHOOTING:
+    - Common issues and solutions
+    - Connection problems
+    - Driver issues
+    - Permission issues (Linux)
+    """
+)
+
+ds_quick_start_guide_instructions = (
+    """
+    Extract information systematically:
+    
+    QUICK START GUIDE:
+    Step-by-step instructions to get debugging working:
+    1. Install drivers
+    2. Connect debugger to target
+    3. Install VS Code extensions
+    4. Create/copy launch.json
+    5. Build project
+    6. Start debugging (F5)
+    """
+)

mfcli/pipeline/analysis/generators/functional_blocks/functional_blocks.py

@@ -0,0 +1,93 @@
+import asyncio
+from asyncio import Semaphore
+from typing import Type, List, Dict
+
+from google.genai.types import File
+from pydantic import BaseModel
+
+from mfcli.client.gemini import Gemini
+from mfcli.crud.functional_blocks import create_functional_blocks
+from mfcli.models.functional_blocks import FunctionalBlocks, FunctionalBlockMeta
+from mfcli.models.pipeline_run import PipelineRun
+from mfcli.pipeline.analysis.generators.functional_blocks.instructions import (
+    fb_extractor_system_instructions,
+    fb_extractor_identify_instructions,
+    fb_extractor_fb_meta_instructions
+)
+from mfcli.pipeline.analysis.generators.functional_blocks.validator import validate_functional_blocks_against_netlist
+from mfcli.utils.logger import get_logger
+from mfcli.utils.orm import Session
+
+logger = get_logger(__name__)
+
+FBExtractedData = FunctionalBlocks | FunctionalBlockMeta
+
+
+class FBCheatSheetGenerator:
+    def __init__(self, gemini: Gemini, files: List[File]):
+        self._gemini = gemini
+        self._sem = Semaphore(5)
+        self._files = files
+
+    async def _extract_fb_data(self, model: Type[BaseModel], instructions: str) -> FBExtractedData:
+        return await self._gemini.generate(
+            prompt=instructions,
+            instructions=fb_extractor_system_instructions,
+            response_model=model,
+            files=self._files
+        )
+
+    async def _retrieve_fb_list(self) -> list[str]:
+        blocks = await self._extract_fb_data(FunctionalBlocks, fb_extractor_identify_instructions)
+        return blocks.functional_blocks
+
+    async def _generate_fb_data(self, name: str) -> FunctionalBlockMeta:
+        block_instructions = fb_extractor_fb_meta_instructions.format(name)
+        return await self._extract_fb_data(FunctionalBlockMeta, block_instructions)
+
+    async def _generate_single_block(self, name: str) -> FunctionalBlockMeta:
+        async with self._sem:
+            return await self._generate_fb_data(name)
+
+    async def generate(self) -> List[FunctionalBlockMeta]:
+        blocks = await self._retrieve_fb_list()
+        tasks = []
+        for block in blocks:
+            tasks.append(self._generate_single_block(block))
+        responses = await asyncio.gather(*tasks)
+        block_data = []
+        for response in responses:
+            if isinstance(response, Exception):
+                continue
+            block_data.append(response)
+        return block_data
+
+
+def _get_functional_blocks_dict(blocks: List[FunctionalBlockMeta]) -> Dict:
+    return {
+        "functional_blocks": [block.model_dump() for block in blocks]
+    }
+
+
+async def extract_functional_blocks_from_file(
+        db: Session,
+        pipeline_run: PipelineRun,
+        gemini: Gemini,
+        schematic_files: List[File] | None = None,
+        user_guide_files: List[File] | None = None,
+        netlist_file: File | None = None
+) -> Dict:
+    if not schematic_files and not user_guide_files and not netlist_file:
+        error = "At least one of these file types must be specified: [schematic, user_guide, netlist]"
+        raise ValueError(error)
+    files = []
+    if schematic_files:
+        files += schematic_files
+    if user_guide_files:
+        files += user_guide_files
+    if netlist_file:
+        files.append(netlist_file)
+    blocks = await FBCheatSheetGenerator(gemini, files).generate()
+    validate_functional_blocks_against_netlist(db, pipeline_run.id, blocks)
+    create_functional_blocks(db, pipeline_run, blocks)
+    return _get_functional_blocks_dict(blocks)

mfcli/pipeline/analysis/generators/functional_blocks/instructions.py

@@ -0,0 +1,34 @@
+fb_extractor_system_instructions = (
+    """
+    You are the Functional Block Generator agent. Your role is to analyze PDF schematics
+    and break them down into functional blocks that describe how different parts of the
+    circuit work together. This information helps firmware developers understand the
+    hardware and write code to support each function.
+
+    A functional block is a group of components that work together for a specific
+    purpose. Common examples include:
+    - Power Supply (voltage regulators, filtering capacitors)
+    - Microcontroller/MCU (main processor and its critical connections)
+    - Communication Interfaces (UART, SPI, I2C, USB, CAN, etc.)
+    - Sensor Interfaces (ADC inputs, sensor power, signal conditioning)
+    - Output Drivers (LED drivers, motor controllers, relay drivers)
+    - Debug/Programming Interface (JTAG, SWD, bootloader pins)
+    - External Connectors (headers, USB ports, JST connectors)
+    - Clock/Crystal circuits
+    - Reset circuits
+    - Protection circuits
+    """
+)
+
+fb_extractor_identify_instructions = "Analyze the schematic to identify functional blocks"
+
+fb_extractor_fb_meta_instructions = (
+    """
+    Extract the following information for the {} functional block:
+
+    BLOCK IDENTIFICATION:
+    - block_name: Clear, descriptive name (e.g., "3.3V Power Supply", "USB Interface")
+    - description: What this block does and why it's important
+    - components: List all reference designators (R1, C1, U1, etc.) in this block
+    """
+)