mfcli 0.2.0__py3-none-any.whl

mfcli/.env.example

@@ -0,0 +1,72 @@
+# Multifactor ADK Backend Configuration
+# 
+# This file contains the configuration settings for mfcli.
+# Copy this file to ~/Multifactor/.env and fill in your API keys.
+#
+# Windows: C:\Users\<username>\Multifactor\.env
+# macOS/Linux: ~/Multifactor/.env
+#
+# Or run: mfcli configure
+# for an interactive setup wizard.
+
+# ============================================================
+# REQUIRED API KEYS
+# ============================================================
+
+# Google Gemini API Key
+# Used for AI-powered document analysis and processing
+# Get your key from: https://aistudio.google.com/app/apikey
+google_api_key=your_google_api_key_here
+
+# OpenAI API Key
+# Used for generating embeddings for RAG (Retrieval-Augmented Generation)
+# Get your key from: https://platform.openai.com/api-keys
+# Note: Requires billing enabled on your OpenAI account
+openai_api_key=your_openai_api_key_here
+
+# LlamaParse API Key
+# Used for advanced PDF parsing and text extraction
+# Get your key from: https://cloud.llamaindex.ai/
+llama_cloud_api_key=your_llamaparse_api_key_here
+
+# DigiKey API Credentials
+# Used for automatic datasheet downloads from DigiKey
+# Get your credentials from: https://developer.digikey.com/
+digikey_client_id=your_digikey_client_id_here
+digikey_client_secret=your_digikey_client_secret_here
+
+# ============================================================
+# VECTOR DATABASE CONFIGURATION
+# ============================================================
+
+# Chunk size for document splitting (number of characters)
+# Larger chunks provide more context but may dilute relevance
+# Default: 1000
+chunk_size=1000
+
+# Overlap between chunks (number of characters)
+# Helps maintain context across chunk boundaries
+# Default: 200
+chunk_overlap=200
+
+# OpenAI embedding model to use
+# Available models: text-embedding-3-small, text-embedding-3-large, text-embedding-ada-002
+# Default: text-embedding-3-small (best balance of cost and quality)
+embedding_model=text-embedding-3-small
+
+# Embedding dimensions
+# text-embedding-3-small: 1536
+# text-embedding-3-large: 3072
+# text-embedding-ada-002: 1536
+embedding_dimensions=1536
+
+# ============================================================
+# OPTIONAL: AWS CONFIGURATION
+# ============================================================
+
+# AWS credentials for S3 storage (optional)
+# Only needed if you want to store files in AWS S3
+# aws_access_key_id=your_aws_access_key
+# aws_secret_access_key=your_aws_secret_key
+# aws_region=us-east-1
+# s3_bucket_name=your_bucket_name

mfcli/agents/controller/agent.py

@@ -0,0 +1,19 @@
+from pathlib import Path
+
+from google.adk.agents import Agent
+
+from mfcli.agents.controller.tools import query_knowledgebase, list_projects
+from mfcli.agents.tools.general import load_agent_config
+from mfcli.pipeline.pipeline import run_pipeline
+
+config_path = Path(__file__).parent / "config.yaml"
+config = load_agent_config(config_path)
+
+root_agent = Agent(
+    name=config.name,
+    model=config.model,
+    description=config.description,
+    instruction=config.instructions,
+    tools=[run_pipeline, query_knowledgebase, list_projects],
+    output_key="pipeline_run_output"
+)

mfcli/agents/controller/config.yaml

@@ -0,0 +1,27 @@
+name: controller_agent
+description: "Controller agent starts the document processing pipeline"
+model: gemini-2.5-flash
+instructions: |
+  You are the controller agent for an engineering document processing pipeline. 
+  You have two main workflows:
+  
+  1. DOCUMENT ANALYSIS WORKFLOW:
+     - The user asks you to analyze documents in a directory
+     - If the user does not specify the name of the project to analyze, run the list_projects tool and ask the user to choose it from the returned list first
+     - Display the list of projects neatly as a bulleted list
+     - Once the user has chosen a project, you MUST then run the run_pipeline tool to start the pipeline
+     - Respond to the user with the output from the pipeline
+     - You will receive a summary from the completed pipeline about the number of files which were successfully processed, the number that failed and the total
+     - You will also receive a list of errors, if any
+     - If there are any error messages analyze and format them for the user, along with file counts and file names
+  
+  2. QUERY WORKFLOW:
+    - The user may ask you general engineering questions
+    - The user may also ask you questions about files which have already been processed
+    - If they ask these types of general questions you will use the query_knowledgebase tool to perform RAG and respond to the user.
+    - You will need the name of the project, run the list_projects tool and ask the user to choose it from the returned list first
+    - Display the list of projects neatly as a bulleted list
+  
+  When to use each workflow:
+  - Use ANALYSIS workflow for: "analyze these files", "process this directory"
+  - Use QUERY workflow for: when the user asks you an engineering-related question or a question about components or files

mfcli/agents/controller/tools.py

@@ -0,0 +1,42 @@
+from typing import List
+
+from mfcli.agents.tools.general import format_error_for_llm
+from mfcli.client.chroma_db import ChromaClient
+from mfcli.crud.project import get_project_by_name
+from mfcli.models.project import Project
+from mfcli.utils.logger import get_logger
+from mfcli.utils.orm import Session
+
+logger = get_logger(__name__)
+
+
+def list_projects() -> List[str]:
+    """
+    Agent will use this tool to list the names of projects.
+    If the user does not supply a project name this tool must be called and the user prompted to select a project.
+    :return: list of project names
+    """
+    with Session() as db:
+        projects: List[Project] = db.query(Project).all()
+        return [project.name for project in projects]
+
+
+def query_knowledgebase(project_name: str, query: str) -> str:
+    """
+    Agent will use this tool to do RAG (query knowledgebase).
+    The user must supply a project_name to select the right knowledgebase.
+    :param project_name:
+    :param query:
+    :return:
+    """
+    with Session() as db:
+        project = get_project_by_name(db, project_name)
+        logger.debug(f"Querying knowledge base for query: {query}")
+        try:
+            chunks = ChromaClient(project.index_id).query(query)
+            return str([{
+                "chunk": chunk.document,
+                "metadata": chunk.metadata
+            } for chunk in chunks])
+        except Exception as e:
+            return format_error_for_llm(e)

mfcli/agents/tools/general.py

@@ -0,0 +1,118 @@
+import json
+import traceback
+from functools import lru_cache
+from pathlib import Path
+from textwrap import dedent
+from typing import Optional, Type
+
+import yaml
+from pydantic import BaseModel
+from typing_extensions import TypeVar
+
+from mfcli.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class AgentConfig(BaseModel):
+    name: str
+    description: str
+    model: str
+    instructions: str
+
+
+@lru_cache
+def load_agent_config(path: str | Path) -> AgentConfig:
+    with open(path, "r") as f:
+        config = yaml.safe_load(f)
+        return AgentConfig(
+            name=config["name"],
+            description=config["description"],
+            model=config["model"],
+            instructions=config["instructions"]
+        )
+
+
+def format_instructions(instructions: str) -> str:
+    return dedent(instructions).strip()
+
+
+def validation_response(validated: bool, e: Exception | str | None = None) -> dict[str:bool | str]:
+    return {"validated": validated, "errors": str(e)}
+
+
+def format_error_for_llm(e: Exception, msg: str | None = None) -> str:
+    return json.dumps({"error": msg or str(e), "stack_trace": traceback.format_exc()})
+
+
+T = TypeVar('T', bound=BaseModel)
+
+
+def validate_schema(model: Type[T], response: str) -> T:
+    return model.model_validate_json(response)
+
+
+def head(file_path: str) -> str:
+    """
+    Get the head of any text file. The agent will run this to determine file subtype.
+    :param file_path: path of the file.
+    :return: first 50 lines of the file.
+    """
+    try:
+        logger.debug(f"Reading: {file_path}")
+        with open(file_path, encoding="utf-8", errors="ignore") as f:
+            return str([line for _, line in zip(range(50), f)])
+    except Exception as e:
+        logger.exception(e)
+        format_error_for_llm(e)
+
+
+# TODO: vectorize_generated_file needs to be updated with project name or chromadb client and placed elsewhere
+def vectorize_generated_file(
+        file_path: str,
+        purpose: str,
+        agent_name: str,
+        project_id: Optional[str] = None
+) -> str:
+    """
+    Vectorize a generated file into the vector database for future RAG queries.
+    
+    This shared tool can be used by any agent to vectorize their output files
+    (PDFs, CSVs, JSON, etc.) into the vector database for context retrieval.
+    
+    Args:
+        file_path: Path to the file to vectorize
+        purpose: Purpose/category of the file (e.g., 'bom', 'errata', 'functional_blocks', 'datasheet')
+        agent_name: Name of the agent generating the file
+        project_id: Optional project ID for metadata
+        
+    Returns:
+        Success message or error details
+    """
+    try:
+        # Import here to avoid circular dependencies
+        from mfcli.utils.datasheet_vectorizer import DatasheetVectorizer
+
+        vectorizer = DatasheetVectorizer()
+
+        # Build metadata
+        additional_metadata = {
+            "agent": agent_name,
+            "generated": True
+        }
+        if project_id:
+            additional_metadata["project_id"] = project_id
+
+        # Vectorize the file
+        vectorizer.vectorize_local_file(
+            file_path=file_path,
+            purpose=purpose,
+            additional_metadata=additional_metadata
+        )
+
+        logger.info(f"Successfully vectorized file: {file_path} (agent: {agent_name}, purpose: {purpose})")
+        return f"Successfully vectorized file: {file_path} with purpose '{purpose}'"
+
+    except Exception as e:
+        logger.error(f"Error vectorizing file {file_path}: {e}")
+        return format_error_for_llm(e)

mfcli/alembic/env.py

@@ -0,0 +1,61 @@
+from logging.config import fileConfig
+
+from sqlalchemy import engine_from_config
+from sqlalchemy import pool
+
+from alembic import context
+
+# import all models to register them
+
+from mfcli.models.base import Base, target_metadata
+
+from mfcli.utils.config import get_config
+from mfcli.utils.orm import get_db_url
+
+# Load Alembic config
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+# if config.config_file_name is not None:
+#     fileConfig(config.config_file_name)
+
+# Dynamically get DB URL (from .env or Pydantic Settings)
+db_url = get_db_url()
+
+# Override the alembic.ini value at runtime
+config.set_main_option("sqlalchemy.url", db_url)
+
+
+def run_migrations_offline():
+    """Run migrations in 'offline' mode."""
+    context.configure(
+        url=db_url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        compare_type=True,
+    )
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online():
+    """Run migrations in 'online' mode."""
+    connectable = engine_from_config(
+        config.get_section(config.config_ini_section, {}),
+        prefix="sqlalchemy.",
+        poolclass=pool.NullPool,
+    )
+
+    with connectable.connect() as connection:
+        context.configure(connection=connection, target_metadata=target_metadata, compare_type=True)
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

mfcli/alembic/script.py.mako

@@ -0,0 +1,28 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision: str = ${repr(up_revision)}
+down_revision: Union[str, Sequence[str], None] = ${repr(down_revision)}
+branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
+depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    ${downgrades if downgrades else "pass"}

mfcli/alembic/versions/6ccc0c7c397c_added_fields_to_pdf_parts_model.py

@@ -0,0 +1,39 @@
+"""Added fields to pdf parts model
+
+Revision ID: 6ccc0c7c397c
+Revises: 769019ef4870
+Create Date: 2025-12-15 22:36:48.423361
+
+"""
+from typing import Sequence, Union
+
+import sqlmodel
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '6ccc0c7c397c'
+down_revision: Union[str, Sequence[str], None] = '769019ef4870'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('pdf_parts', sa.Column('start_page', sa.Integer(), nullable=True))
+    op.add_column('pdf_parts', sa.Column('end_page', sa.Integer(), nullable=True))
+    op.add_column('pdf_parts', sa.Column('title', sqlmodel.sql.sqltypes.AutoString(), nullable=True))
+    op.add_column('pdf_parts', sa.Column('section_no', sa.Integer(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('pdf_parts', 'section_no')
+    op.drop_column('pdf_parts', 'title')
+    op.drop_column('pdf_parts', 'end_page')
+    op.drop_column('pdf_parts', 'start_page')
+    # ### end Alembic commands ###

mfcli/alembic/versions/769019ef4870_added_gemini_file_path_to_pdf_part_model.py

@@ -0,0 +1,33 @@
+"""Added gemini file path to pdf part model
+
+Revision ID: 769019ef4870
+Revises: 7d5adb2a47a7
+Create Date: 2025-12-13 01:12:10.436014
+
+"""
+from typing import Sequence, Union
+
+import sqlmodel
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '769019ef4870'
+down_revision: Union[str, Sequence[str], None] = '7d5adb2a47a7'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('pdf_parts', sa.Column('gemini_file_id', sqlmodel.sql.sqltypes.AutoString(length=40), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('pdf_parts', 'gemini_file_id')
+    # ### end Alembic commands ###

mfcli/alembic/versions/7a2e3a779fdc_added_functional_block_and_component_.py

@@ -0,0 +1,54 @@
+"""added functional block and component models
+
+Revision ID: 7a2e3a779fdc
+Revises: 4c0509f2a4cf
+Create Date: 2025-12-10 01:09:06.904237
+
+"""
+from typing import Sequence, Union
+
+import sqlmodel
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '7a2e3a779fdc'
+down_revision: Union[str, Sequence[str], None] = '7fcb7d6a5836'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('functional_blocks',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('pipeline_run_id', sa.Integer(), nullable=False),
+    sa.Column('name', sqlmodel.sql.sqltypes.AutoString(length=255), nullable=False),
+    sa.Column('description', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.ForeignKeyConstraint(['pipeline_run_id'], ['pipeline_runs.id'], ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_functional_blocks_name'), 'functional_blocks', ['name'], unique=False)
+    op.create_index(op.f('ix_functional_blocks_pipeline_run_id'), 'functional_blocks', ['pipeline_run_id'], unique=False)
+    op.create_table('functional_block_components',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('functional_block_id', sa.Integer(), nullable=False),
+    sa.Column('ref', sqlmodel.sql.sqltypes.AutoString(length=45), nullable=False),
+    sa.ForeignKeyConstraint(['functional_block_id'], ['functional_blocks.id'], ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_functional_block_components_functional_block_id'), 'functional_block_components', ['functional_block_id'], unique=False)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f('ix_functional_block_components_functional_block_id'), table_name='functional_block_components')
+    op.drop_table('functional_block_components')
+    op.drop_index(op.f('ix_functional_blocks_pipeline_run_id'), table_name='functional_blocks')
+    op.drop_index(op.f('ix_functional_blocks_name'), table_name='functional_blocks')
+    op.drop_table('functional_blocks')
+    # ### end Alembic commands ###

mfcli/alembic/versions/7d5adb2a47a7_added_pdf_parts_model.py

@@ -0,0 +1,41 @@
+"""Added PDF parts model
+
+Revision ID: 7d5adb2a47a7
+Revises: 7a2e3a779fdc
+Create Date: 2025-12-12 23:27:28.310305
+
+"""
+from typing import Sequence, Union
+
+import sqlmodel
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '7d5adb2a47a7'
+down_revision: Union[str, Sequence[str], None] = '7a2e3a779fdc'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('pdf_parts',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('file_id', sa.Integer(), nullable=False),
+    sa.Column('path', sqlmodel.sql.sqltypes.AutoString(length=600), nullable=False),
+    sa.ForeignKeyConstraint(['file_id'], ['files.id'], ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_pdf_parts_file_id'), 'pdf_parts', ['file_id'], unique=False)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f('ix_pdf_parts_file_id'), table_name='pdf_parts')
+    op.drop_table('pdf_parts')
+    # ### end Alembic commands ###