mfcli 0.2.0__py3-none-any.whl

mfcli/pipeline/analysis/generators/mcu_errata/instructions.py

@@ -0,0 +1,77 @@
+errata_extraction_base_instructions = (
+    """
+    You are the MCU Errata Cheat Sheet Generator agent. Your role is to analyze MCU errata
+    documents (PDF files) and extract ONLY firmware-relevant issues - silicon bugs that can be
+    addressed or worked around in firmware code. Exclude hardware-only issues that cannot be
+    fixed in software. You will be given an MCU Errata file to analyze.
+    """
+)
+
+extract_errata_ids_instruction = (
+    f"""
+    {errata_extraction_base_instructions}
+
+    Your job is to extract all the Errata IDs. Return a list of official IDs from document (e.g., "I2C_01", "ADV0123")
+  """
+)
+
+extract_errata_instructions = (
+    f"""
+    {errata_extraction_base_instructions}
+
+    You be given the errata ID to extract, and you will extract this info:
+
+    a. IDENTIFICATION:
+        - errata_id: Official ID from document (e.g., "I2C_01", "ADV0123")
+        - title: Brief descriptive title
+        - affected_modules: List of affected peripherals/modules
+        Examples: ["I2C", "SPI", "UART", "ADC", "Timer", "DMA", "RTC"]
+    b. SEVERITY CLASSIFICATION:
+        - Critical: Can cause data corruption, system hang, or major malfunction
+        - Major: Significant functional impact, workaround is complex
+        - Minor: Minor inconvenience, easy workaround
+    c. DETAILED INFORMATION:
+        - description: Clear explanation of the bug
+        - conditions: When/how the bug occurs
+        * Specific register values
+        * Timing conditions
+        * Operating modes
+        * Environmental conditions (temperature, voltage)
+    d. FIRMWARE WORKAROUND:
+        - firmware_workaround: Specific code-level workaround
+        Examples:
+        * "Add 10us delay after setting register X"
+        * "Avoid using bits [7:5] in CONFIG register"
+        * "Initialize peripheral in specific order: Step 1, Step 2, Step 3"
+        * "Use polling instead of interrupts for this peripheral"
+        * "Apply calibration value from factory settings"
+        - Be SPECIFIC - provide actual steps/code guidance
+    e. IMPACT:
+        - impact: How this affects firmware operation
+        Examples:
+        * "May cause I2C communication failures"
+        * "Incorrect ADC readings below 10% of range"
+        * "System hang if DMA used with this peripheral"
+        f. SILICON REVISIONS:
+        - affected_revisions: Which chip revisions have this bug
+        Examples: ["Rev A", "Rev B"], ["All revisions"], ["Rev 1.0 - 1.2"]
+    """
+)
+
+errata_document_summary_instructions = (
+    f"""
+    {errata_extraction_base_instructions}
+
+    You will extract three items from this document:
+
+    1. Errata document name, for example Silicon Errata Rev 1.2 - March 2024.
+
+    2. MCU name, for example MSPM0L1306
+
+    3. Top-level firmware recommendations. 
+    Examples:
+    * "Always use polling mode for I2C on Rev A silicon"
+    * "Add delays in ADC initialization sequence"
+    * "Avoid simultaneous use of Timer3 and DMA Channel 2"
+    """
+)

mfcli/pipeline/analysis/generators/mcu_errata/mcu_errata.py

@@ -0,0 +1,95 @@
+import asyncio
+from asyncio import Semaphore
+from collections import defaultdict
+from typing import Dict, List
+
+from google.genai.types import File as GeminiFile
+
+from mfcli.models.file import File
+from mfcli.models.mcu_errata import ErrataIDs, ErrataItem, ErrataTopLevelSummary
+from mfcli.pipeline.analysis.generators.generator_base import GeneratorBase
+from mfcli.pipeline.analysis.generators.mcu_errata.instructions import (
+    extract_errata_ids_instruction,
+    extract_errata_instructions,
+    errata_document_summary_instructions
+)
+from mfcli.pipeline.run_context import PipelineRunContext
+from mfcli.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class ErrataCheatSheetGenerator(GeneratorBase):
+    def __init__(self, context: PipelineRunContext, db_file: File, uploads: List[GeminiFile]):
+        super().__init__(context, db_file, uploads)
+
+    async def _extract_errata_ids(self) -> list[str]:
+        prompt = "Use the Errata file to extract errata_ids"
+        errata_ids: ErrataIDs = await self._context.gemini.generate(
+            prompt=prompt,
+            instructions=extract_errata_ids_instruction,
+            response_model=ErrataIDs,
+            files=self._uploads
+        )
+        return errata_ids.ids
+
+    async def _extract_errata(self, errata_id: str, sem: Semaphore) -> ErrataItem:
+        prompt = f"Extract errata info from ID: {errata_id}"
+        async with sem:
+            return await self._context.gemini.generate(
+                prompt=prompt,
+                instructions=extract_errata_instructions,
+                response_model=ErrataItem,
+                files=self._uploads
+            )
+
+    async def _generate_top_level_summary(self) -> ErrataTopLevelSummary:
+        prompt = "Generate top-level document summary"
+        return await self._context.gemini.generate(
+            prompt=prompt,
+            instructions=errata_document_summary_instructions,
+            response_model=ErrataTopLevelSummary,
+            files=self._uploads
+        )
+
+    async def _create_summary(self, errata: list[ErrataItem]) -> Dict:
+        critical = []
+        major = []
+        minor = []
+        model_issues_counts = defaultdict(int)
+        for erratum in errata:
+            if erratum.severity == 'Critical':
+                critical.append(erratum.model_dump())
+            elif erratum.severity == 'Major':
+                major.append(erratum.model_dump())
+            else:
+                minor.append(erratum.model_dump())
+            for module in erratum.affected_modules:
+                model_issues_counts[module] += 1
+        summary = await self._generate_top_level_summary()
+        return {
+            "errata_cheat_sheet": {
+                "mcu_name": summary.mcu_name,
+                "errata_document": summary.errata_document,
+                "total_issues": len(errata),
+                "critical_issues": critical,
+                "major_issues": major,
+                "minor_issues": minor,
+                "summary_by_module": model_issues_counts,
+                "key_recommendations": summary.recommendations
+            }
+        }
+
+    async def generate(self) -> Dict:
+        errata_ids = await self._extract_errata_ids()
+        errata: list[ErrataItem] = []
+        sem = asyncio.Semaphore(5)
+        tasks = [self._extract_errata(errata_id, sem) for errata_id in errata_ids]
+        results: list[ErrataItem | Exception] = await asyncio.gather(*tasks, return_exceptions=True)
+        for result in results:
+            if isinstance(result, Exception):
+                logger.exception(result)
+                logger.error(f"Error extracting errata")
+                continue
+            errata.append(result)
+        return await self._create_summary(errata)

mfcli/pipeline/analysis/generators/summary/summary.py

@@ -0,0 +1,47 @@
+from typing import List, Dict
+
+from google.genai.types import File as GeminiFile
+
+from mfcli.models.file import File
+from mfcli.models.llm_response import LLMResponse
+from mfcli.models.pdf_parts import PDFPart
+from mfcli.pipeline.analysis.generators.generator_base import GeneratorBase
+from mfcli.pipeline.run_context import PipelineRunContext
+
+user_guide_summary_instructions = (
+    """
+    You will receive sections of a hardware engineering user guide PDF. 
+    Your job is to summarize those sections. 
+    You will also receive the sections that have been summarized so far. 
+    Use the sections that have been summarized as context to generate new summaries. 
+    ONLY output the summary text, no other information.
+    The summary text will be read directly by users. 
+    """
+)
+
+
+class SummaryCheatSheetGenerator(GeneratorBase):
+    def __init__(self, context: PipelineRunContext, db_file: File, uploads: List[GeminiFile]):
+        super().__init__(context, db_file, uploads)
+
+    async def generate(self) -> Dict:
+        pdf_parts: List[PDFPart] = self._file.pdf_parts
+        pdf_parts.sort(key=lambda part: part.section_no)
+        summaries = []
+        for pdf_part in pdf_parts:
+            prompt = f"Summarize the {pdf_part.title} section\n\nCurrent summaries:\n\n{summaries}"
+            upload = self._context.gemini_file_cache[pdf_part.gemini_file_id]
+            response: LLMResponse = await self._context.gemini.generate(
+                prompt=prompt,
+                instructions=user_guide_summary_instructions,
+                response_model=LLMResponse,
+                files=[upload]
+            )
+            summaries.append({
+                "no": pdf_part.section_no,
+                "title": pdf_part.title,
+                "summary": response.text
+            })
+        return {
+            "summaries": summaries
+        }

mfcli/pipeline/classifier.py

@@ -0,0 +1,93 @@
+import hashlib
+import mimetypes
+import os
+from pathlib import Path
+
+import pandas as pd
+
+from werkzeug.utils import secure_filename
+
+from mfcli.constants.file_types import SupportedFileTypes, FileTypes
+from mfcli.models.file_metadata import FileMetadata
+from mfcli.utils.logger import get_logger
+
+MAX_FILE_GB = 1
+MAX_FILE_SIZE = 1024 * 1024 * 1024 * MAX_FILE_GB
+
+logger = get_logger(__name__)
+
+
+def is_csv(file_path):
+    pd.read_csv(file_path, nrows=5, encoding_errors='ignore')
+
+
+def validate_file(metadata: FileMetadata):
+    logger.debug(f"Validating file: {metadata.name}")
+    try:
+        if metadata.type_id == FileTypes.CSV:
+            is_csv(metadata.path)
+        else:
+            logger.debug(f"File type has no validator: {metadata.type_name}")
+    except Exception as e:
+        raise ValueError(f"The file is not a valid {metadata.type_name} file: {e}")
+
+
+def get_file_metadata(file_path: str, is_datasheet: bool) -> tuple[FileMetadata, bytes]:
+    logger.debug(f"Starting categorize_and_validate_file tool: {file_path}")
+    file_name = os.path.basename(file_path)
+    file_name = secure_filename(file_name).lower().strip()
+
+    file_ext = os.path.splitext(file_name)[1]
+    file_type_name = file_ext.replace('.', '').upper()
+    if not file_type_name:
+        raise ValueError("File requires an extension")
+    if file_type_name not in SupportedFileTypes:
+        raise ValueError(f"File extension is not supported: {file_type_name}")
+    file_type_id = FileTypes[file_type_name].value
+    logger.debug(f"File type id: {file_type_id}")
+
+    path = Path(file_path)
+    if not path.exists():
+        raise ValueError("File does not exist")
+    if not os.access(path, os.R_OK):
+        raise ValueError("File is not readable")
+
+    file_bytes = os.stat(file_path).st_size
+    if file_bytes == 0:
+        raise ValueError(f"File is empty: {file_name}")
+    if file_bytes > MAX_FILE_SIZE:
+        raise ValueError(f"File size exceeds limit: {file_bytes}")
+
+    logger.debug(f"File validated: {file_path}")
+    with open(file_path, "rb") as fp:
+        content = fp.read()
+        md5_sum = hashlib.md5(content).hexdigest()
+        
+        # Use mimetypes module to guess MIME type from file extension
+        mime_type, _ = mimetypes.guess_type(file_path)
+        
+        # If mimetypes can't detect it, use default based on file type
+        if mime_type is None:
+            # Use the first supported MIME type as default
+            mime_types = SupportedFileTypes[file_type_name]["mime_types"]
+            mime_type = list(mime_types)[0] if mime_types else 'application/octet-stream'
+        
+        # Validate MIME type matches expected types for this file extension
+        if mime_type not in SupportedFileTypes[file_type_name]["mime_types"]:
+            logger.warning(f"Extension {file_type_name} has unexpected MIME type: {mime_type}. Expected one of: {SupportedFileTypes[file_type_name]['mime_types']}")
+            # Use the first supported MIME type as default instead of failing
+            mime_type = list(SupportedFileTypes[file_type_name]["mime_types"])[0]
+        
+        file_metadata = FileMetadata(
+            name=file_name,
+            size=file_bytes,
+            md5=md5_sum,
+            path=file_path,
+            mime=mime_type,
+            ext=file_ext,
+            type_id=file_type_id,
+            type_name=file_type_name,
+            is_datasheet=is_datasheet
+        )
+        logger.debug(file_metadata)
+        return file_metadata, content

mfcli/pipeline/data_enricher.py

@@ -0,0 +1,15 @@
+from typing import TypeVar
+
+from sqlmodel import SQLModel
+
+from mfcli.client.chroma_db import ChromaClient
+from mfcli.constants.file_types import FileSubtypes
+from mfcli.utils.datasheet_vectorizer import get_datasheets_for_bom_entries
+from mfcli.utils.orm import Session
+
+T = TypeVar('T', bound=SQLModel)
+
+
+async def enrich_data_for_model(db: Session, chroma_db: ChromaClient, subtype: int, instances: list[T]):
+    if subtype == FileSubtypes.BOM:
+        await get_datasheets_for_bom_entries(db, chroma_db, instances)

mfcli/pipeline/extractor.py

@@ -0,0 +1,34 @@
+from mfcli.constants.file_types import FileTypes
+from mfcli.models.file import File
+from mfcli.pipeline.extractors.pdf import extract_text_from_pdf
+from mfcli.utils.files import is_text_mime_type
+
+import os.path
+
+from mfcli.client.llama_parse import LlamaParseClient
+
+
+class TextExtractor(LlamaParseClient):
+    def __init__(self):
+        super().__init__()
+
+    @staticmethod
+    def extract_pdf_bytes(pdf_bytes: bytes):
+        return extract_text_from_pdf(pdf_bytes)
+
+    def extract_text_from_file_bytes(self, file_name: str, file_bytes: bytes) -> str:
+        return self.parse(file_name, file_bytes)
+
+    def extract_text(self, file_path: str):
+        with open(file_path, "rb") as f:
+            file_name = os.path.basename(file_path)
+            file_bytes = f.read()
+            return self.parse(file_name, file_bytes)
+
+
+def extract_document_text(file: File, file_bytes: bytes) -> str:
+    if is_text_mime_type(file.mime_type):
+        return file_bytes.decode(errors='ignore')
+    elif file.type == FileTypes.PDF:
+        return TextExtractor().extract_pdf_bytes(file_bytes)
+    raise ValueError(f"Unsupported MIME type: {file.mime_type}")

mfcli/pipeline/extractors/pdf.py

@@ -0,0 +1,12 @@
+import fitz
+from io import BytesIO
+
+
+def extract_text_from_pdf(pdf_bytes: bytes) -> str:
+    """Extract all text from a PDF given as bytes (no temp file needed)."""
+    text = []
+    # Open from a memory stream instead of file path
+    with fitz.open(stream=BytesIO(pdf_bytes), filetype="pdf") as doc:
+        for page in doc:
+            text.append(page.get_text("text"))
+    return "\n".join(text)

mfcli/pipeline/parser.py

@@ -0,0 +1,120 @@
+import json
+from pathlib import Path
+from typing import Type
+
+import pandas as pd
+from sqlmodel import SQLModel
+
+from mfcli.constants.file_types import FileTypes, FileSubtypes
+from mfcli.crud.netlist import create_netlist
+from mfcli.models.file import File
+from mfcli.models.netlist import Netlist
+from mfcli.pipeline.parsers.netlist.kicad_legacy_net import parse_kicad_legacy_net_file
+from mfcli.pipeline.parsers.netlist.kicad_spice import parse_kicad_spice_file
+from mfcli.pipeline.parsers.netlist.pads import parse_pads_file
+from mfcli.pipeline.parsers.netlist.protel import parse_protel_file
+from mfcli.pipeline.schema_mapper import SchemaMappings, SubtypeModels
+from mfcli.utils.logger import get_logger
+from mfcli.utils.orm import Session
+
+logger = get_logger(__name__)
+
+
+def parse_csv(file_path: str) -> dict:
+    logger.debug(f"Parsing CSV: {file_path}")
+    df = pd.read_csv(file_path, header='infer', encoding_errors='ignore')
+    json_str = df.to_json(orient="records")
+    logger.debug(f"CSV parsed: {file_path}")
+    return json.loads(json_str)
+
+
+def _extract_schema_from_csv(
+        file: File,
+        input_column_field_map: dict[str, str],
+        model: Type[SQLModel]
+) -> list[SQLModel]:
+    rows = parse_csv(file.path)
+    model_instances: list[SQLModel] = []
+    for row in rows:
+        try:
+            mapped_data = {}
+            for input_col, model_field in input_column_field_map.items():
+                if input_col in row:
+                    mapped_data[model_field] = row[input_col]
+                else:
+                    mapped_data[model_field] = None
+            instance = model(**mapped_data)
+            instance.file_id = file.id
+            model_instances.append(instance)
+            logger.debug(f"Model parsed from CSV: {instance}")
+        except Exception as e:
+            logger.warn(e)
+    if not model_instances:
+        raise ValueError(f"No data could be parsed from this CSV: {file.path}")
+    return model_instances
+
+
+class SchemaParser:
+    def __init__(self, db: Session, file: File):
+        self._db = db
+        self.file = file
+
+    def _parse_with_schema_mappings(
+            self,
+            model: Type[SQLModel],
+            mappings: SchemaMappings
+    ) -> list[SQLModel]:
+        input_column_field_map = {mapping.input_field: mapping.mapped_field for mapping in mappings.fields}
+        logger.debug(f"Model: {model}")
+        if self.file.type == FileTypes.CSV:
+            return _extract_schema_from_csv(self.file, input_column_field_map, model)
+        raise ValueError(f"Unsupported extraction file type: {self.file.type}")
+
+    @staticmethod
+    def _is_netlist_file(subtype: FileSubtypes):
+        if subtype in [
+            FileSubtypes.KICAD_LEGACY_NET,
+            FileSubtypes.KICAD_SPICE,
+            FileSubtypes.PADS_PCB_ASCII,
+            FileSubtypes.PROTEL_ALTIUM
+        ]:
+            return True
+        return False
+
+    def _parse_netlist_file(self, subtype: FileSubtypes, file_path: Path) -> Netlist:
+        if subtype == FileSubtypes.KICAD_LEGACY_NET:
+            netlist_schema = parse_kicad_legacy_net_file(file_path)
+        elif subtype == FileSubtypes.PADS_PCB_ASCII:
+            netlist_schema = parse_pads_file(file_path)
+        elif subtype == FileSubtypes.KICAD_SPICE:
+            netlist_schema = parse_kicad_spice_file(file_path)
+        elif subtype == FileSubtypes.PROTEL_ALTIUM:
+            netlist_schema = parse_protel_file(file_path)
+        else:
+            raise ValueError(f"Netlist file has no parser: {self.file.name}")
+        netlist = create_netlist(self.file.pipeline_run_id, netlist_schema)
+        return netlist
+
+    def _parse_without_schema_mappings(self) -> list[SQLModel]:
+        subtype = self.file.sub_type
+        file_path = Path(self.file.path)
+        if self._is_netlist_file(subtype):
+            return [self._parse_netlist_file(subtype, file_path)]
+        raise ValueError(f"No parser for file subtype: {self.file.sub_type}")
+
+    def parse(self, mappings: SchemaMappings | None) -> list[SQLModel]:
+        logger.debug(f"Extracting schema from file: {self.file.name}")
+        if not SubtypeModels.get(self.file.sub_type):
+            raise ValueError(f"Cannot find subtype model for subtype: {self.file.sub_type}")
+        model: Type[SQLModel] = SubtypeModels.get(self.file.sub_type)
+        if mappings:
+            instances = self._parse_with_schema_mappings(model, mappings)
+        else:
+            instances = self._parse_without_schema_mappings()
+        self._db.add_all(instances)
+        logger.debug(f"File has been successfully parsed")
+        return instances
+
+
+def parse_schema(db: Session, file: File, mappings: SchemaMappings | None) -> list[SQLModel]:
+    return SchemaParser(db, file).parse(mappings)

mfcli/pipeline/parsers/netlist/edif.py

@@ -0,0 +1,93 @@
+import re
+from collections import defaultdict
+from typing import Dict, Any
+
+
+def parse(file_path: str) -> Dict[str, Any]:
+    """
+    Parse an EDIF file and extract a relational JSON representation:
+    component-pin-net mapping, deduplicated net names, annotated with voltage domains and pull-up/pull-down info.
+    """
+    with open(file_path, "r") as f:
+        edif_text = f.read()
+
+    # --- Step 1: Extract NETLIST_TEXT ---
+    match = re.search(r'\(property NETLIST_TEXT \(string "(.*?)"\)\)', edif_text, re.DOTALL)
+    if not match:
+        raise ValueError("No NETLIST_TEXT property found in EDIF")
+
+    netlist_text = match.group(1)
+    netlist_text = netlist_text.replace('\\n', '\n').strip()
+
+    # --- Step 2: Parse SPICE-style lines ---
+    component_pattern = re.compile(
+        r'^(?P<name>\w+)\s+(?P<n1>\S+)\s+(?P<n2>\S+)\s+(?P<rest>.*)$', re.MULTILINE
+    )
+
+    components = []
+    nets = defaultdict(lambda: {"name": None, "connected_pins": []})
+
+    for match in component_pattern.finditer(netlist_text):
+        name = match.group("name")
+        n1, n2 = match.group("n1"), match.group("n2")
+        rest = match.group("rest")
+
+        # Deduce type from prefix (SPICE convention)
+        type_prefix = name[0].lower()
+        type_map = {
+            "r": "resistor",
+            "c": "capacitor",
+            "l": "inductor",
+            "v": "voltage_source",
+            "i": "current_source",
+            "e": "op_amp",
+            "d": "diode",
+            "j": "jumper",
+            "led": "led",
+            "s": "switch",
+            "y": "crystal",
+            "sh": "shunt",
+            "rt": "thermistor",
+            "tp": "transistor",
+        }
+        comp_type = type_map.get(type_prefix, "unknown")
+
+        pins = [n1, n2]
+        components.append({
+            "name": name,
+            "type": comp_type,
+            "pins": pins,
+            "params": rest,
+        })
+
+        # Connect pins to nets
+        for pin in pins:
+            nets[pin]["name"] = pin
+            nets[pin]["connected_pins"].append({"component": name, "pin": pin})
+
+    # --- Step 3: Deduplicate net names ---
+    unique_nets = list(nets.values())
+
+    # --- Step 4: Annotate nets ---
+    for net in unique_nets:
+        name = net["name"].lower()
+        if name in ("vcc", "vdd", "vin"):
+            net["voltage_domain"] = "high"
+        elif name in ("gnd", "vss", "0"):
+            net["voltage_domain"] = "ground"
+        else:
+            net["voltage_domain"] = "signal"
+
+        # Simple pull heuristic
+        if "pullup" in name:
+            net["pull"] = "pull-up"
+        elif "pulldown" in name:
+            net["pull"] = "pull-down"
+        else:
+            net["pull"] = None
+
+    # --- Step 5: Return structured output ---
+    return {
+        "components": components,
+        "nets": unique_nets
+    }