modaic 0.2.0__py3-none-any.whl → 0.4.0__py3-none-any.whl

modaic/agents/rag_agent.py

@@ -1,4 +1,4 @@
-from typing import List
+from typing import Any
 
 from modaic import Indexer, PrecompiledAgent, PrecompiledConfig
 from modaic.context import Context
@@ -19,7 +19,10 @@ class RAGAgentConfig(PrecompiledConfig):
 
 @builtin_indexer(agent_name)
 class RAGIndexer(Indexer):
-    def ingest(self, config: RAGAgentConfig, contexts: List[Context]):
+    def __init__(self, config: RAGAgentConfig):
+        super().__init__(config)
+
+    def index(self, contents: Any):
         pass
 
 

modaic/databases/sql_database.py

@@ -5,7 +5,19 @@ from typing import Any, Callable, Iterable, List, Literal, Optional, Tuple
 from urllib.parse import urlencode
 
 import pandas as pd
-from sqlalchemy import Column, CursorResult, MetaData, String, Text, create_engine, inspect, text
+from sqlalchemy import (
+    JSON,
+    Column,
+    CursorResult,
+    Index,
+    MetaData,
+    PrimaryKeyConstraint,
+    String,
+    Text,
+    create_engine,
+    inspect,
+    text,
+)
 from sqlalchemy import Table as SQLTable
 from sqlalchemy.dialects import sqlite
 from sqlalchemy.orm import sessionmaker
@@ -81,30 +93,43 @@ class SQLiteBackend(SQLDatabaseBackend):
 
 
 class SQLDatabase:
+    METADATA_TABLE_NAME = "modaic_metadata"
+
     def __init__(
         self,
         backend: SQLDatabaseBackend | str,
         engine_kwargs: dict = None,  # TODO: This may not be a smart idea, may want to enforce specific kwargs
         session_kwargs: dict = None,  # TODO: This may not be a smart idea, may want to enforce specific kwargs
+        track_metadata: bool = False,
     ):
         self.url = backend.url if isinstance(backend, SQLDatabaseBackend) else backend
         self.engine = create_engine(self.url, **(engine_kwargs or {}))
         self.metadata = MetaData()
         self.session = sessionmaker(bind=self.engine, **(session_kwargs or {}))
         self.inspector = inspect(self.engine)
-        self.preparer = IdentifierPreparer(sqlite.dialect())
+        self.preparer = self.engine.dialect.identifier_preparer
 
         # Create metadata table to store table metadata
-        self.metadata_table = SQLTable(
-            "metadata",
-            self.metadata,
-            Column("table_name", String(255), primary_key=True),
-            Column("metadata_json", Text),
+        if track_metadata:
+            self._ensure_metadata_table()
+        self.metadata.reflect(bind=self.engine)
+        self.metadata_table: Optional[Table] = (
+            self.metadata.tables[self.METADATA_TABLE_NAME] if track_metadata else None
         )
-        self.metadata.create_all(self.engine)
         self.connection = None
         self._in_transaction = False
 
+    def _ensure_metadata_table(self) -> None:
+        """Create the metadata table if missing."""
+        if not self.inspector.has_table(self.METADATA_TABLE_NAME):
+            SQLTable(
+                self.METADATA_TABLE_NAME,
+                self.metadata,
+                Column("table_name", String(255), primary_key=True),
+                Column("metadata_json", Text),
+            )
+            self.metadata.create_all(self.engine)
+
     def add_table(
         self,
         table: BaseTable,
@@ -115,17 +140,17 @@ class SQLDatabase:
         with self.connect() as connection:
             # Use the connection for to_sql to respect transaction context
             table._df.to_sql(table.name, connection, if_exists=if_exists, index=False)
-
-            # Remove existing metadata for this table if it exists
-            connection.execute(self.metadata_table.delete().where(self.metadata_table.c.table_name == table.name))
-
-            # Insert new metadata
-            connection.execute(
-                self.metadata_table.insert().values(
-                    table_name=table.name,
-                    metadata_json=json.dumps(table.metadata),
+            if self.metadata_table is not None:
+                # Remove existing metadata for this table if it exists
+                connection.execute(self.metadata_table.delete().where(self.metadata_table.c.table_name == table.name))
+
+                # Insert new metadata
+                connection.execute(
+                    self.metadata_table.insert().values(
+                        table_name=table.name,
+                        metadata_json=json.dumps(table.metadata),
+                    )
                 )
-            )
             if self._should_commit():
                 connection.commit()
 
@@ -151,7 +176,8 @@ class SQLDatabase:
             command = text(f"DROP TABLE {if_exists} {safe_name}")
             connection.execute(command)
             # Also remove metadata for this table
-            connection.execute(self.metadata_table.delete().where(self.metadata_table.c.table_name == name))
+            if self.metadata_table is not None:
+                connection.execute(self.metadata_table.delete().where(self.metadata_table.c.table_name == name))
             if self._should_commit():
                 connection.commit()
 
@@ -197,6 +223,10 @@ class SQLDatabase:
         Returns:
             Dictionary containing the table's metadata, or empty dict if not found.
         """
+        if self.metadata_table is None:
+            raise ValueError(
+                "Metadata table is not enabled. Please enable metadata tracking when initializing the SQLDatabase. with track_metadata=True."
+            )
         with self.connect() as connection:
             result = connection.execute(
                 self.metadata_table.select().where(self.metadata_table.c.table_name == name)

modaic/exceptions.py

@@ -36,3 +36,12 @@ class BackendCompatibilityError(ModaicError):
     """Raised when a feature is not supported by a backend"""
 
     pass
+
+
+class MissingSecretError(AuthenticationError):
+    """Raised when a secret is missing"""
+
+    def __init__(self, message: str, secret_name: str):
+        self.message = message
+        self.secret_name = secret_name
+        super().__init__(message)

modaic/module_utils.py

@@ -243,6 +243,12 @@ def init_agent_repo(repo_path: str, with_code: bool = True) -> Path:
         if src_init.exists() and not dest_init.exists():
             shutil.copy2(src_init, dest_init)
 
+    for extra_file in get_extra_files():
+        if extra_file.is_dir():
+            shutil.copytree(extra_file, repo_dir / extra_file.relative_to(project_root))
+        else:
+            shutil.copy2(extra_file, repo_dir / extra_file.relative_to(project_root))
+
     return repo_dir
 
 
@@ -272,23 +278,52 @@ def get_ignored_files() -> list[Path]:
     pyproject_path = Path("pyproject.toml")
     doc = tomlk.parse(pyproject_path.read_text(encoding="utf-8"))
 
-    # Safely get [tool.modaic.ignore]
-    ignore_table = (
+    # Safely get [tool.modaic.exclude]
+    files = (
         doc.get("tool", {})  # [tool]
         .get("modaic", {})  # [tool.modaic]
-        .get("ignore")  # [tool.modaic.ignore]
+        .get("exclude", {})  # [tool.modaic.exclude]
+        .get("files", [])  # [tool.modaic.exclude] files = ["file1", "file2"]
     )
 
-    if ignore_table is None or "files" not in ignore_table:
-        return []
+    excluded: list[Path] = []
+    for entry in files:
+        entry = Path(entry)
+        if not entry.is_absolute():
+            entry = project_root / entry
+        if entry.exists():
+            excluded.append(entry)
+    return excluded
 
-    ignored: list[Path] = []
-    for entry in ignore_table["files"]:
-        try:
-            ignored.append((project_root / entry).resolve())
-        except OSError:
-            continue
-    return ignored
+
+def get_extra_files() -> list[Path]:
+    """Return a list of extra files that should be excluded from staging."""
+    project_root = resolve_project_root()
+    pyproject_path = Path("pyproject.toml")
+    doc = tomlk.parse(pyproject_path.read_text(encoding="utf-8"))
+    files = (
+        doc.get("tool", {})  # [tool]
+        .get("modaic", {})  # [tool.modaic]
+        .get("include", {})  # [tool.modaic.include]
+        .get("files", [])  # [tool.modaic.include] files = ["file1", "file2"]
+    )
+    included: list[Path] = []
+    for entry in files:
+        entry = Path(entry)
+        if entry.is_absolute():
+            try:
+                entry = entry.resolve()
+                entry.relative_to(project_root.resolve())
+            except ValueError:
+                warnings.warn(
+                    f"{entry} will not be bundled because it is not inside the current working directory", stacklevel=4
+                )
+        else:
+            entry = project_root / entry
+        if entry.resolve().exists():
+            included.append(entry)
+
+    return included
 
 
 def create_pyproject_toml(repo_dir: Path, package_name: str):
@@ -304,7 +339,7 @@ def create_pyproject_toml(repo_dir: Path, package_name: str):
     if "project" not in doc_old:
         raise KeyError("No [project] table in old TOML")
     doc_new["project"] = doc_old["project"]
-    doc_new["project"]["dependencies"] = get_filtered_dependencies(doc_old["project"]["dependencies"])
+    doc_new["project"]["dependencies"] = get_final_dependencies(doc_old["project"]["dependencies"])
     if "tool" in doc_old and "uv" in doc_old["tool"] and "sources" in doc_old["tool"]["uv"]:
         doc_new["tool"] = {"uv": {"sources": doc_old["tool"]["uv"]["sources"]}}
         warn_if_local(doc_new["tool"]["uv"]["sources"])
@@ -315,29 +350,32 @@ def create_pyproject_toml(repo_dir: Path, package_name: str):
         tomlk.dump(doc_new, fp)
 
 
-def get_filtered_dependencies(dependencies: list[str]) -> list[str]:
+def get_final_dependencies(dependencies: list[str]) -> list[str]:
     """
     Get the dependencies that should be included in the bundled agent.
+    Filters out "[tool.modaic.ignore] dependencies. Adds [tool.modaic.include] dependencies.
     """
     pyproject_path = Path("pyproject.toml")
     doc = tomlk.parse(pyproject_path.read_text(encoding="utf-8"))
 
-    # Safely get [tool.modaic.ignore]
-    ignore_table = (
+    # Safely get [tool.modaic.exclude]
+    exclude_deps = (
         doc.get("tool", {})  # [tool]
         .get("modaic", {})  # [tool.modaic]
-        .get("ignore", {})  # [tool.modaic.ignore]
+        .get("exclude", {})  # [tool.modaic.exclude]
+        .get("dependencies", [])  # [tool.modaic.exclude] dependencies = ["praw", "sagemaker"]
+    )
+    include_deps = (
+        doc.get("tool", {})  # [tool]
+        .get("modaic", {})  # [tool.modaic]
+        .get("include", {})  # [tool.modaic.include]
+        .get("dependencies", [])  # [tool.modaic.include] dependencies = ["praw", "sagemaker"]
     )
 
-    if "dependencies" not in ignore_table:
-        return dependencies
-
-    ignored_dependencies = ignore_table["dependencies"]
-    if not ignored_dependencies:
-        return dependencies
-    pattern = re.compile(r"\b(" + "|".join(map(re.escape, ignored_dependencies)) + r")\b")
-    filtered_dependencies = [pkg for pkg in dependencies if not pattern.search(pkg)]
-    return filtered_dependencies
+    if exclude_deps:
+        pattern = re.compile(r"\b(" + "|".join(map(re.escape, exclude_deps)) + r")\b")
+        dependencies = [pkg for pkg in dependencies if not pattern.search(pkg)]
+    return dependencies + include_deps
 
 
 def warn_if_local(sources: dict[str, dict]):

modaic/precompiled.py

@@ -2,12 +2,13 @@ import inspect
 import json
 import os
 import pathlib
+import warnings
 from abc import ABC, abstractmethod
 from pathlib import Path
 from typing import (
     TYPE_CHECKING,
+    Any,
     Dict,
-    List,
     Optional,
     Type,
     TypeVar,
@@ -15,11 +16,13 @@ from typing import (
 )
 
 import dspy
+from git import config
 from pydantic import BaseModel
 
 from modaic.module_utils import create_agent_repo
 from modaic.observability import Trackable, track_modaic_obj
 
+from .exceptions import MissingSecretError
 from .hub import load_repo, push_folder_to_hub
 from .module_utils import _module_path
 
@@ -128,6 +131,7 @@ class PrecompiledConfig(BaseModel):
         return self.model_dump_json()
 
 
+# Use a metaclass to enforce super().__init__() with config
 class PrecompiledAgent(dspy.Module):
     """
     Bases: `dspy.Module`
@@ -234,15 +238,25 @@ class PrecompiledAgent(dspy.Module):
                 extra_auto_classes["AutoRetriever"] = self.retriever
         self.config.save_precompiled(path, extra_auto_classes)
         self.save(path / "agent.json")
+        _clean_secrets(path / "agent.json")
 
     @classmethod
-    def from_precompiled(cls: Type[A], path: str | Path, config_options: Optional[dict] = None, **kwargs) -> A:
+    def from_precompiled(
+        cls: Type[A],
+        path: str | Path,
+        config_options: Optional[dict] = None,
+        api_key: Optional[str | dict[str, str]] = None,
+        hf_token: Optional[str | dict[str, str]] = None,
+        **kwargs,
+    ) -> A:
         """
         Loads the agent and the config from the given path.
 
         Args:
             path: The path to load the agent and config from. Can be a local path or a path on Modaic Hub.
             config_options: A dictionary containg key-value pairs used to override the default config.
+            api_key: Your API key.
+            hf_token: Your Hugging Face token.
             **kwargs: Additional keyword arguments forwarded to the PrecompiledAgent's constructor.
 
         Returns:
@@ -260,7 +274,9 @@ class PrecompiledAgent(dspy.Module):
         agent = cls(config, **kwargs)
         agent_state_path = local_dir / "agent.json"
         if agent_state_path.exists():
-            agent.load(agent_state_path)
+            secrets = {"api_key": api_key, "hf_token": hf_token}
+            state = _get_state_with_secrets(agent_state_path, secrets)
+            agent.load_state(state)
         return agent
 
     def push_to_hub(
@@ -374,7 +390,7 @@ class Indexer(Retriever):
     config: PrecompiledConfig
 
     @abstractmethod
-    def ingest(self, contexts: List["Context"], **kwargs):
+    def index(self, contents: Any, **kwargs):
         pass
 
 
@@ -404,11 +420,88 @@ def _push_to_hub(
 
 def is_local_path(s: str | Path) -> bool:
     # absolute or relative filesystem path
+    if isinstance(s, Path):
+        return True
     s = str(s)
+
+    print("SSSS", s)
     if os.path.isabs(s) or s.startswith((".", "/", "\\")):
         return True
     parts = s.split("/")
     # hub IDs: "repo" or "user/repo"
-    if len(parts) == 1 or (len(parts) == 2 and all(parts)):
+    if len(parts) == 1:
+        raise ValueError(
+            f"Invalid repo: '{s}'. Please prefix local paths with './', '/', or '../' . And use 'user/repo' format for hub paths."
+        )
+    elif len(parts) == 2 and all(parts):
         return False
     return True
+
+
+SECRET_MASK = "********"
+COMMON_SECRETS = ["api_key", "hf_token"]
+
+
+def _clean_secrets(path: Path, extra_secrets: Optional[list[str]] = None):
+    """
+    Removes all secret keys from `lm` dict in agent.json file
+    """
+    secret_keys = COMMON_SECRETS + (extra_secrets or [])
+
+    with open(path, "r") as f:
+        d = json.load(f)
+
+    for predictor in d.values():
+        lm = predictor.get("lm", None)
+        if lm is None:
+            continue
+        for k in lm.keys():
+            if k in secret_keys:
+                lm[k] = SECRET_MASK
+
+    with open(path, "w") as f:
+        json.dump(d, f, indent=2)
+
+
+def _get_state_with_secrets(path: Path, secrets: dict[str, str | dict[str, str] | None]):
+    """`
+    Fills secret keys in `lm` dict in agent.json file
+
+    Args:
+        path: The path to the agent.json file.
+        secrets: A dictionary containing the secrets to fill in the `lm` dict.
+            - Dict[k,v] where k is the name of a secret (e.g. "api_key") and v is the value of the secret
+            - If v is a string, every lm will use v for k
+            - if v is a dict, each key of v should be the name of a named predictor
+            (e.g. "my_module.predict", "my_module.summarizer") mapping to the secret value for that predictor
+    Returns:
+        A dictionary containing the state of the agent.json file with the secrets filled in.
+    """
+    with open(path, "r") as f:
+        named_predictors = json.load(f)
+
+    def _get_secret(predictor_name: str, secret_name: str) -> Optional[str]:
+        if secret_val := secrets.get(secret_name):
+            if isinstance(secret_val, str):
+                return secret_val
+            elif isinstance(secret_val, dict):
+                return secret_val.get(predictor_name)
+        return None
+
+    for predictor_name, predictor in named_predictors.items():
+        lm = predictor.get("lm", {})
+        for kw, arg in lm.items():
+            if kw in COMMON_SECRETS and arg != "" and arg != SECRET_MASK:
+                warnings.warn(
+                    f"{str(path)} exposes the secret key {kw}. Please remove it or ensure this file is not made public."
+                )
+            secret = _get_secret(predictor_name, kw)
+            if secret is not None and arg != "" and arg != SECRET_MASK:
+                raise ValueError(
+                    f"Failed to fill insert secret value for {predictor_name}['lm']['{kw}']. It is already set to {arg}"
+                )
+            elif secret is None and kw in COMMON_SECRETS:
+                raise MissingSecretError(f"Please specify a value for {kw} in the secrets dictionary", kw)
+            elif secret is not None:
+                lm[kw] = secret
+    return named_predictors

{modaic-0.2.0.dist-info → modaic-0.4.0.dist-info}/METADATA

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: modaic
-Version: 0.2.0
-Summary: Modular Agent Infrastructure Collective, a python framework for managing and sharing DSPy agents
+Version: 0.4.0
+Summary: Modular Agent Infrastructure Collection, a python framework for managing and sharing DSPy agents
 Author-email: Tyrin <tytodd@mit.edu>, Farouk <farouk@modaic.dev>
 License: MIT License
         

{modaic-0.2.0.dist-info → modaic-0.4.0.dist-info}/RECORD

@@ -1,16 +1,16 @@
 modaic/__init__.py,sha256=xHu2SUk3OMvb8PIzrVCRS1pBk-Ho9BhwmzKOf_bOjGc,809
 modaic/auto.py,sha256=rPOdQ7s-YGBQLa_v6lVONH8pbOrarPwp4VzRErp0y5c,9091
 modaic/datasets.py,sha256=K-PpPSYIxJI0-yH-SBVpk_EfCM9i_uPz-brmlzP7hzI,513
-modaic/exceptions.py,sha256=XxzxOWjZTzT3l1BqTr7coJnVGxJq53uppRNrqP__YGo,651
+modaic/exceptions.py,sha256=Vq-eWEPSiqA3G3HaI2uZ_3Uwg4aIvulDnyOFAZc1FBk,903
 modaic/hub.py,sha256=d5HQjaE26K1qNCBc32qJtrpESyRv6OiniAteasiN_rk,11290
 modaic/indexing.py,sha256=VdILiXiLVzgV1pSTV8Ho7x1dZtd31Y9z60d_Qtqr2NU,4195
-modaic/module_utils.py,sha256=oJUOddvNGEIvEABOf7rzMB9erOkjDPm7FWLaBJ0XTSA,11797
+modaic/module_utils.py,sha256=I6kGCmGSuHM9lxz8rpCIkdFHCh0M7OIyw3GZ3966YWY,13442
 modaic/observability.py,sha256=LgR4gJM4DhD-xlVX52mzRQSPgLQzbeh2LYPmQVqSh-A,9947
-modaic/precompiled.py,sha256=cLRNoDB_ivawGftDpWQbkjjThrtID7bG1E-XVxi5kEM,14804
+modaic/precompiled.py,sha256=7__DgYaVXFE7xZR6_jtRAVxUw7v0E59zcPWlGfs7raU,18303
 modaic/query_language.py,sha256=BJIigR0HLapiIn9fF7jM7PkLM8OWUDjwYuxmzcCVvyo,9487
 modaic/types.py,sha256=gcx8J4oxrHwxA7McyYV4OKHsuPhhmowJtJIgjJQbLto,10081
 modaic/utils.py,sha256=doJs-XL4TswSQFBINZeKrik-cvjZk-tS9XmWH8fOYiw,794
-modaic/agents/rag_agent.py,sha256=f8s3EILOPUxMpOKDoAvk-cfLE8S9kFNvkEcAC5z2EmQ,798
+modaic/agents/rag_agent.py,sha256=h17YQHXGRPMFn_liUDRNFYc2m0U5KXZ16Q0UELCqmEE,844
 modaic/agents/registry.py,sha256=z6GuPxGrq2dinCamiMJ_HVPsD9Tp9XWDUSMZ-uhWPrU,2446
 modaic/context/__init__.py,sha256=FK-bxSu36yGFF1rATy4Yzl4Fpv9kYOlRpBRfr_4moiM,560
 modaic/context/base.py,sha256=x66_lcdQ063DiluC6UnFEH4etgJkbAyukrgrp2KLV5U,40368
@@ -19,7 +19,7 @@ modaic/context/table.py,sha256=9Lh2_UyK3OsWmgUfZQa8jDeVAPAk_iueT89_cruDeSo,17699
 modaic/context/text.py,sha256=gCVQx15FrPcmpr2GYkJca6noh7fw_nTcCt-hISwcnvQ,2581
 modaic/databases/__init__.py,sha256=-w_yiY-Sqi1SgcPD5oAQL7MU4VXTihPa1GYGlrHfsFw,784
 modaic/databases/graph_database.py,sha256=j44PgWGMeD3dtPZe5sRpKnruTA3snm_TiXncusTzqIQ,9990
-modaic/databases/sql_database.py,sha256=wqy7AqsalhmYsbNPy0FCAg1FrUKN6Bd8ytwyJireC94,12057
+modaic/databases/sql_database.py,sha256=62h9GtZv8gEAo1qHOQsxPL9bJPwweNmWLcRjYcpFIMY,13037
 modaic/databases/vector_database/__init__.py,sha256=sN1SuSAMC9NHJDOa80BN_olccaHgmiW2Ek57hBvdZWo,306
 modaic/databases/vector_database/vector_database.py,sha256=RsuRemgFV06opY26CekqLLRoAEFYOGl_CMuFETrYS0c,25238
 modaic/databases/vector_database/benchmarks/baseline.py,sha256=ZhiYzHnizsLesAIQA93RhWaaYxEZp6ygExmnBhU9Dio,5209
@@ -33,8 +33,8 @@ modaic/databases/vector_database/vendors/qdrant.py,sha256=AbpHGcgLb-kRsJGnwFEktk
 modaic/storage/__init__.py,sha256=Zs-Y_9jfYUE8XVp8z-El0ZXFM_ZVMqM9aQ6fgGPZsf8,131
 modaic/storage/file_store.py,sha256=kSS7gTP_-16wR3Xgq3frF1BZ8Dw8N--kG4V9rrCXPcc,7315
 modaic/storage/pickle_store.py,sha256=fu9jkmmKNE852Y4R1NhOFePLfd2gskhHSXxuq1G1S3I,778
-modaic-0.2.0.dist-info/licenses/LICENSE,sha256=7LMx9j453Vz1DoQbFot8Uhp9SExF5wlOx7c8vw2qhsE,1333
-modaic-0.2.0.dist-info/METADATA,sha256=71CN4H1DdXoN4f-ahbju8JqUaTU4zagjm-E__nIwesY,8651
-modaic-0.2.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-modaic-0.2.0.dist-info/top_level.txt,sha256=RXWGuF-TsW8-17DveTJMPRiAgg_Rf2mq5F3R7tNu6t8,7
-modaic-0.2.0.dist-info/RECORD,,
+modaic-0.4.0.dist-info/licenses/LICENSE,sha256=7LMx9j453Vz1DoQbFot8Uhp9SExF5wlOx7c8vw2qhsE,1333
+modaic-0.4.0.dist-info/METADATA,sha256=g7EbTl48wysK-lb6XnfBspOb85iH3oQXmllD-Tnplls,8651
+modaic-0.4.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+modaic-0.4.0.dist-info/top_level.txt,sha256=RXWGuF-TsW8-17DveTJMPRiAgg_Rf2mq5F3R7tNu6t8,7
+modaic-0.4.0.dist-info/RECORD,,