modaic 0.1.2__py3-none-any.whl → 0.3.0__py3-none-any.whl

modaic/__init__.py

@@ -1,4 +1,4 @@
-from .auto_agent import AutoAgent, AutoConfig, AutoRetriever
+from .auto import AutoAgent, AutoConfig, AutoRetriever
 from .indexing import Embedder
 from .observability import Trackable, configure, track, track_modaic_obj
 from .precompiled import Indexer, PrecompiledAgent, PrecompiledConfig, Retriever
@@ -22,4 +22,16 @@ __all__ = [
     "Prop",
     "Value",
     "parse_modaic_filter",
+    "Condition",
 ]
+_configured = False
+
+
+def _auto_configure():
+    global _configured
+    if not _configured:
+        configure()
+        _configured = True
+
+
+_auto_configure()

modaic/{auto_agent.py → auto.py}

@@ -4,24 +4,38 @@ import os
 import sys
 from functools import lru_cache
 from pathlib import Path
-from typing import Literal, Optional, Type
+from typing import Callable, Literal, Optional, Type, TypedDict
 
-from .hub import load_repo
+from .hub import AGENTS_CACHE, load_repo
 from .precompiled import PrecompiledAgent, PrecompiledConfig, Retriever, is_local_path
 
 MODAIC_TOKEN = os.getenv("MODAIC_TOKEN")
 
 
-_REGISTRY = {}  # maps model_type string -> (ConfigCls, ModelCls)
+class RegisteredRepo(TypedDict, total=False):
+    AutoConfig: Type[PrecompiledConfig]
+    AutoAgent: Type[PrecompiledAgent]
+    AutoRetriever: Type[Retriever]
 
 
-def register(model_type: str, config_cls: Type[PrecompiledConfig], model_cls: Type[PrecompiledAgent]):
-    _REGISTRY[model_type] = (config_cls, model_cls)
+_REGISTRY: dict[str, RegisteredRepo] = {}
 
 
+def register(
+    name: str,
+    auto_type: Literal["AutoConfig", "AutoAgent", "AutoRetriever"],
+    cls: Type[PrecompiledConfig | PrecompiledAgent | Retriever],
+):
+    if name in _REGISTRY:
+        _REGISTRY[name][auto_type] = cls
+    else:
+        _REGISTRY[name] = {auto_type: cls}
+
+
+# TODO: Cleanup code still using parent_mdoule
 @lru_cache
 def _load_dynamic_class(
-    repo_dir: str, class_path: str, parent_module: Optional[str] = None
+    repo_dir: Path, class_path: str, hub_path: str = None
 ) -> Type[PrecompiledConfig | PrecompiledAgent | Retriever]:
     """
     Load a class from a given repository directory and fully qualified class path.
@@ -29,27 +43,24 @@ def _load_dynamic_class(
     Args:
       repo_dir: Absolute path to a local repository directory containing the code.
       class_path: Dotted path to the target class (e.g., "pkg.module.Class").
-      parent_module: Optional dotted module prefix (e.g., "swagginty.TableRAG"). If provided,
-                     class_path is treated as relative to this module and only the agents cache
-                     root is added to sys.path.
+      hub_path: The path to the repo on modaic hub (if its a hub repo) *Must be specified if its a hub repo*
 
     Returns:
       The resolved class object.
     """
-
-    repo_path = Path(repo_dir)
-
-    repo_dir_str = str(repo_path)
-    print(f"repo_dir_str: {repo_dir_str}")
-    print(f"sys.path: {sys.path}")
-    if repo_dir_str not in sys.path:
-        # print(f"Inserting {repo_dir_str} into sys.path")
-        sys.path.insert(0, repo_dir_str)
-    full_path = (
-        f"{parent_module}.{class_path}"
-        if parent_module and not class_path.startswith(parent_module + ".")
-        else class_path
-    )
+    if hub_path is None:
+        # Local folder case
+        repo_dir_str = str(repo_dir)
+        if repo_dir_str not in sys.path:
+            sys.path.insert(0, repo_dir_str)
+        full_path = f"{class_path}"
+    else:
+        # loaded hub repo case
+        agents_cache_str = str(AGENTS_CACHE)
+        if agents_cache_str not in sys.path:
+            sys.path.insert(0, agents_cache_str)
+        parent_module = hub_path.replace("/", ".")
+        full_path = f"{parent_module}.{class_path}"
 
     module_name, _, attr = full_path.rpartition(".")
     module = importlib.import_module(module_name)
@@ -62,27 +73,31 @@ class AutoConfig:
     """
 
     @staticmethod
-    def from_precompiled(repo_path: str, *, parent_module: Optional[str] = None, **kwargs) -> PrecompiledConfig:
+    def from_precompiled(repo_path: str, **kwargs) -> PrecompiledConfig:
+        local = is_local_path(repo_path)
+        repo_dir = load_repo(repo_path, local)
+        return AutoConfig._from_precompiled(repo_dir, hub_path=repo_path if not local else None, **kwargs)
+
+    @staticmethod
+    def _from_precompiled(repo_dir: Path, hub_path: str = None, **kwargs) -> PrecompiledConfig:
         """
         Load a config for an agent or retriever from a precompiled repo.
 
         Args:
-          repo_path: Hub path ("user/repo") or a local directory.
-          parent_module: Optional dotted module prefix (e.g., "swagginty.TableRAG") to use to import classes from repo_path. If provided, overides default parent_module behavior.
+          repo_dir: The path to the repo directory. the loaded local repository directory.
+          hub_path: The path to the repo on modaic hub (if its a hub repo) *Must be specified if its a hub repo*
 
         Returns:
           A config object constructed via the resolved config class.
         """
-        local = is_local_path(repo_path)
-        repo_dir = load_repo(repo_path, local)
 
         cfg_path = repo_dir / "config.json"
         if not cfg_path.exists():
-            raise FileNotFoundError(f"Failed to load AutoConfig, config.json not found in {repo_path}")
+            raise FileNotFoundError(f"Failed to load AutoConfig, config.json not found in {hub_path or str(repo_dir)}")
         with open(cfg_path, "r") as fp:
             cfg = json.load(fp)
 
-        ConfigClass = _load_auto_class(repo_path, repo_dir, "AutoConfig", parent_module=parent_module)  # noqa: N806
+        ConfigClass = _load_auto_class(repo_dir, "AutoConfig", hub_path=hub_path)  # noqa: N806
         return ConfigClass(**{**cfg, **kwargs})
 
 
@@ -96,7 +111,6 @@ class AutoAgent:
         repo_path: str,
         *,
         config_options: Optional[dict] = None,
-        parent_module: Optional[str] = None,
         project: Optional[str] = None,
         **kw,
     ) -> PrecompiledAgent:
@@ -105,23 +119,25 @@ class AutoAgent:
 
         Args:
           repo_path: Hub path ("user/repo") or local directory.
-          parent_module: Optional dotted module prefix (e.g., "swagginty.TableRAG") to use to import classes from repo_path. If provided, overides default parent_module behavior.
           project: Optional project name. If not provided and repo_path is a hub path, defaults to the repo name.
           **kw: Additional keyword arguments forwarded to the Agent constructor.
 
         Returns:
           An instantiated Agent subclass.
         """
+        # TODO: fast lookups via registry
         local = is_local_path(repo_path)
         repo_dir = load_repo(repo_path, local)
+        hub_path = repo_path if not local else None
 
         if config_options is None:
             config_options = {}
 
-        cfg = AutoConfig.from_precompiled(repo_dir, local=True, parent_module=parent_module, **config_options)
-        AgentClass = _load_auto_class(repo_path, repo_dir, "AutoAgent", parent_module=parent_module)  # noqa: N806
+        cfg = AutoConfig._from_precompiled(repo_dir, hub_path=hub_path, **config_options)
+        AgentClass = _load_auto_class(repo_dir, "AutoAgent", hub_path=hub_path)  # noqa: N806
 
         # automatically configure repo and project from repo_path if not provided
+        # TODO: redundant checks in if statement. Investigate removing.
         if not local and "/" in repo_path and not repo_path.startswith("/"):
             parts = repo_path.split("/")
             if len(parts) >= 2:
@@ -146,7 +162,6 @@ class AutoRetriever:
         repo_path: str,
         *,
         config_options: Optional[dict] = None,
-        parent_module: Optional[str] = None,
         project: Optional[str] = None,
         **kw,
     ) -> Retriever:
@@ -155,7 +170,6 @@ class AutoRetriever:
 
         Args:
           repo_path: hub path ("user/repo"), or local directory.
-          parent_module: Optional dotted module prefix (e.g., "swagginty.TableRAG") to use to import classes from repo_path. If provided, overides default parent_module behavior.
           project: Optional project name. If not provided and repo_path is a hub path, defaults to the repo name.
           **kw: Additional keyword arguments forwarded to the Retriever constructor.
 
@@ -164,14 +178,16 @@ class AutoRetriever:
         """
         local = is_local_path(repo_path)
         repo_dir = load_repo(repo_path, local)
+        hub_path = repo_path if not local else None
 
         if config_options is None:
             config_options = {}
 
-        cfg = AutoConfig.from_precompiled(repo_dir, local=True, parent_module=parent_module, **config_options)
-        RetrieverClass = _load_auto_class(repo_path, repo_dir, "AutoRetriever", parent_module=parent_module)  # noqa: N806
+        cfg = AutoConfig._from_precompiled(repo_dir, hub_path=hub_path, **config_options)
+        RetrieverClass = _load_auto_class(repo_dir, "AutoRetriever", hub_path=hub_path)  # noqa: N806
 
         # automatically configure repo and project from repo_path if not provided
+        # TODO: redundant checks in if statement. Investigate removing.
         if not local and "/" in repo_path and not repo_path.startswith("/"):
             parts = repo_path.split("/")
             if len(parts) >= 2:
@@ -186,27 +202,25 @@ class AutoRetriever:
 
 
 def _load_auto_class(
-    repo_path: str,
     repo_dir: Path,
     auto_name: Literal["AutoConfig", "AutoAgent", "AutoRetriever"],
-    parent_module: Optional[str] = None,
+    hub_path: str = None,
 ) -> Type[PrecompiledConfig | PrecompiledAgent | Retriever]:
     """
     Load a class from the auto_classes.json file.
 
     Args:
-        repo_path: The path to the repo. (local or hub path)
         repo_dir: The path to the repo directory. the loaded local repository directory.
         auto_name: The name of the auto class to load. (AutoConfig, AutoAgent, AutoRetriever)
-        parent_module: The parent module to use to import the class.
+        hub_path: The path to the repo on modaic hub (if its a hub repo) *Must be specified if its a hub repo*
     """
     # determine if the repo was loaded from local or hub
-    local = is_local_path(repo_path)
+    local = hub_path is None
     auto_classes_path = repo_dir / "auto_classes.json"
 
     if not auto_classes_path.exists():
         raise FileNotFoundError(
-            f"Failed to load {auto_name}, auto_classes.json not found in {repo_path}, if this is your repo, make sure you push_to_hub() with `with_code=True`"
+            f"Failed to load {auto_name}, auto_classes.json not found in {hub_path or str(repo_dir)}, if this is your repo, make sure you push_to_hub() with `with_code=True`"
         )
 
     with open(auto_classes_path, "r") as fp:
@@ -214,15 +228,33 @@ def _load_auto_class(
 
     if not (auto_class_path := auto_classes.get(auto_name)):
         raise KeyError(
-            f"{auto_name} not found in {repo_path}/auto_classes.json. Please check that the auto_classes.json file is correct."
+            f"{auto_name} not found in {hub_path or str(repo_dir)}/auto_classes.json. Please check that the auto_classes.json file is correct."
         ) from None
 
-    if auto_class_path in _REGISTRY:
-        _, LoadedClass = _REGISTRY[auto_class_path]  # noqa: N806
-    else:
-        if parent_module is None and not local:
-            parent_module = str(repo_path).replace("/", ".")
-
-        repo_dir = repo_dir.parent.parent if not local else repo_dir
-        LoadedClass = _load_dynamic_class(repo_dir, auto_class_path, parent_module=parent_module)  # noqa: N806
+    repo_dir = repo_dir.parent.parent if not local else repo_dir
+    LoadedClass = _load_dynamic_class(repo_dir, auto_class_path, hub_path=hub_path)  # noqa: N806
     return LoadedClass
+
+
+def builtin_agent(name: str) -> Callable[[Type], Type]:
+    def _wrap(cls: Type) -> Type:
+        register(name, "AutoAgent", cls)
+        return cls
+
+    return _wrap
+
+
+def builtin_indexer(name: str) -> Callable[[Type], Type]:
+    def _wrap(cls: Type) -> Type:
+        register(name, "AutoRetriever", cls)
+        return cls
+
+    return _wrap
+
+
+def builtin_config(name: str) -> Callable[[Type], Type]:
+    def _wrap(cls: Type) -> Type:
+        register(name, "AutoConfig", cls)
+        return cls
+
+    return _wrap

modaic/context/__init__.py

@@ -12,7 +12,7 @@ from .table import (
     Table,
     TableFile,
 )
-from .text import Text
+from .text import Text, TextFile
 
 __all__ = [
     "MultiTabbedTable",
@@ -31,4 +31,5 @@ __all__ = [
     "Prop",
     "HydratedAttr",
     "requires_hydration",
+    "TextFile",
 ]

modaic/context/base.py

@@ -22,7 +22,7 @@ from pydantic._internal._model_construction import ModelMetaclass
 from pydantic.fields import ModelPrivateAttr
 from pydantic.main import IncEx
 from pydantic.v1 import Field as V1Field
-from pydantic_core import CoreSchema, SchemaSerializer
+from pydantic_core import SchemaSerializer
 
 from ..query_language import Prop
 from ..storage.file_store import FileStore

modaic/context/table.py

@@ -38,7 +38,7 @@ class BaseTable(Context, ABC):
         """
         Return up to 3 distinct sample values from the given column.
 
-        Picks at most three unique, non-null, short (<64 chars) values from
+        Picks at most three unique, non-null, short (&lt;64 chars) values from
         the column, favoring speed by sampling after de-duplicating values.
 
         Args:

modaic/context/text.py

@@ -47,7 +47,7 @@ class TextFile(Context):
     file_type: Literal["txt"] = "txt"
 
     def hydrate(self, file_store: FileStore) -> None:
-        file = file_store.get(self.file_ref)
+        file = file_store.get(self.file_ref).file
         if isinstance(file, Path):
             file = file.read_text()
         else:
@@ -91,4 +91,4 @@ class TextFile(Context):
                 chunks.append(Text(text=chunk))
             return chunks
 
-        self.apply_to_chunks(chunk_text_fn)
+        self.chunk_with(chunk_text_fn)

modaic/databases/graph_database.py

@@ -12,12 +12,13 @@ from typing import (
     Type,
 )
 
-from dotenv import load_dotenv
+from dotenv import find_dotenv, load_dotenv
 
 from ..context.base import Context, Relation
 from ..observability import Trackable, track_modaic_obj
 
-load_dotenv()
+env_file = find_dotenv(usecwd=True)
+load_dotenv(env_file)
 
 
 if TYPE_CHECKING:

modaic/datasets.py

@@ -0,0 +1,22 @@
+import dspy
+
+
+class Dataset:
+    def __init__(self, data: list):
+        self.data = data
+
+    def to_dspy(self) -> list:
+        return dspy.Dataset(self.data)
+
+    @classmethod
+    def from_csv(cls, file_path: str) -> "Dataset":
+        with open(file_path, "r") as file:
+            data = file.read()
+        return cls(data)
+
+    @classmethod
+    def from_hub(cls, dataset_name: str) -> "Dataset":
+        from datasets import load_dataset
+
+        data = load_dataset(dataset_name)
+        return cls(data)

modaic/hub.py

@@ -1,15 +1,17 @@
 import os
+import shutil
 from pathlib import Path
 from typing import Any, Dict, Optional
 
 import git
 import requests
-from dotenv import load_dotenv
+from dotenv import find_dotenv, load_dotenv
 
 from .exceptions import AuthenticationError, RepositoryExistsError, RepositoryNotFoundError
 from .utils import compute_cache_dir
 
-load_dotenv()
+env_file = find_dotenv(usecwd=True)
+load_dotenv(env_file)
 
 MODAIC_TOKEN = os.getenv("MODAIC_TOKEN")
 MODAIC_GIT_URL = os.getenv("MODAIC_GIT_URL", "git.modaic.dev").replace("https://", "").rstrip("/")
@@ -80,6 +82,7 @@ def create_remote_repo(repo_path: str, access_token: str, exist_ok: bool = False
         raise Exception(f"Request failed: {str(e)}") from e
 
 
+# FIXME: make faster. Currently takes ~9 seconds
 def push_folder_to_hub(
     folder: str,
     repo_path: str,
@@ -123,10 +126,11 @@ def push_folder_to_hub(
             "Modaic fast paths not yet implemented. Please load agents with 'user/repo' or 'org/repo' format"
         )
     assert repo_path.count("/") <= 1, f"Extra '/' in repo_path: {repo_path}"
-
+    # TODO: try pushing first and on error create the repo. create_remote_repo currently takes ~1.5 seconds to run
     create_remote_repo(repo_path, access_token, exist_ok=True)
     username = get_user_info(access_token)["login"]
 
+    # FIXME: takes 6 seconds
     try:
         # 1) If local folder is not a git repository, initialize it.
         local_repo = git.Repo.init(folder)
@@ -205,6 +209,7 @@ def get_repo_payload(repo_name: str) -> Dict[str, Any]:
     return payload
 
 
+# TODO: add persistent filesystem based cache mapping access_token to user_info. Currently takes ~1 second
 def get_user_info(access_token: str) -> Dict[str, Any]:
     """
     Returns the user info for the given access token.
@@ -214,12 +219,14 @@ def get_user_info(access_token: str) -> Dict[str, Any]:
         access_token: The access token to get the user info for.
 
     Returns:
+    ```python
         {
             "login": str,
             "email": str,
             "avatar_url": str,
             "name": str,
         }
+    ```
     """
     global user_info
     if user_info:
@@ -243,6 +250,7 @@ def get_user_info(access_token: str) -> Dict[str, Any]:
     return user_info
 
 
+# TODO:
 def git_snapshot(
     repo_path: str,
     *,
@@ -265,7 +273,6 @@ def git_snapshot(
     elif access_token is None:
         raise ValueError("Access token is required")
 
-    # If a local folder path is provided, just return it
     repo_dir = Path(AGENTS_CACHE) / repo_path
     username = get_user_info(access_token)["login"]
     try:
@@ -291,10 +298,26 @@ def git_snapshot(
         repo.git.reset("--hard", f"origin/{target}")
         return repo_dir
     except Exception as e:
-        repo_dir.rmdir()
+        shutil.rmtree(repo_dir)
         raise e
 
 
+def _move_to_commit_sha_folder(repo: git.Repo) -> git.Repo:
+    """
+    Moves the repo to a new path based on the commit SHA. (Unused for now)
+    Args:
+        repo: The git.Repo object.
+
+    Returns:
+        The new git.Repo object.
+    """
+    commit = repo.head.commit
+    repo_dir = Path(repo.working_dir)
+    new_path = repo_dir / commit.hexsha
+    repo_dir.rename(new_path)
+    return git.Repo(new_path)
+
+
 def load_repo(repo_path: str, is_local: bool = False) -> Path:
     if is_local:
         path = Path(repo_path)

modaic/indexing.py

@@ -77,7 +77,7 @@ class PineconeReranker(Reranker):
         try:
             from pinecone import Pinecone
         except ImportError:
-            raise ImportError("Pinecone is not installed. Please install it with `uv add pinecone`")
+            raise ImportError("Pinecone is not installed. Please install it with `uv add pinecone`") from None
 
         if api_key is None:
             self.pinecone = Pinecone(os.getenv("PINECONE_API_KEY"))

modaic/module_utils.py

@@ -16,6 +16,7 @@ from .utils import compute_cache_dir
 MODAIC_CACHE = compute_cache_dir()
 AGENTS_CACHE = Path(MODAIC_CACHE) / "agents"
 EDITABLE_MODE = os.getenv("EDITABLE_MODE", "false").lower() == "true"
+TEMP_DIR = Path(MODAIC_CACHE) / "temp"
 
 
 def is_builtin(module_name: str) -> bool:
@@ -76,6 +77,7 @@ def is_builtin_or_frozen(mod: ModuleType) -> bool:
     return (name in sys.builtin_module_names) or (origin in ("built-in", "frozen"))
 
 
+# FIXME: make faster. Currently takes ~.70 seconds
 def get_internal_imports() -> Dict[str, ModuleType]:
     """Return only internal modules currently loaded in sys.modules.
 
@@ -107,6 +109,10 @@ def get_internal_imports() -> Dict[str, ModuleType]:
         if is_builtin_or_frozen(module):
             continue
 
+        # edge case: local modaic package
+        if name == "modaic" or "modaic." in name:
+            continue
+
         module_file = getattr(module, "__file__", None)
         if not module_file:
             continue
@@ -189,8 +195,11 @@ def is_external_package(path: Path) -> bool:
 
 def init_agent_repo(repo_path: str, with_code: bool = True) -> Path:
     """Create a local repository staging directory for agent modules and files, excluding ignored files and folders."""
-    repo_dir = Path(AGENTS_CACHE) / repo_path
-    repo_dir.mkdir(parents=True, exist_ok=True)
+    repo_dir = TEMP_DIR / repo_path
+    shutil.rmtree(repo_dir, ignore_errors=True)
+    repo_dir.mkdir(parents=True, exist_ok=False)
+
+    project_root = resolve_project_root()
 
     internal_imports = get_internal_imports()
     ignored_paths = get_ignored_files()
@@ -207,12 +216,12 @@ def init_agent_repo(repo_path: str, with_code: bool = True) -> Path:
     if not with_code:
         return repo_dir
 
-    for module_name, module in internal_imports.items():
-        module_file = getattr(module, "__file__", None)
+    for _, module in internal_imports.items():
+        module_file = Path(getattr(module, "__file__", None))
         if not module_file:
             continue
         try:
-            src_path = Path(module_file).resolve()
+            src_path = module_file.resolve()
         except OSError:
             continue
         if src_path.suffix != ".py":
@@ -223,25 +232,33 @@ def init_agent_repo(repo_path: str, with_code: bool = True) -> Path:
             continue
         seen_files.add(src_path)
 
-        # Split modul_name to get the relative path
-        name_parts = module_name.split(".")
-        if src_path.name == "__init__.py":
-            copy_module_layout(repo_dir, name_parts)
-            dest_path = repo_dir.joinpath(*name_parts) / "__init__.py"
-        else:
-            if len(name_parts) > 1:
-                copy_module_layout(repo_dir, name_parts[:-1])
-            else:
-                repo_dir.mkdir(parents=True, exist_ok=True)
-            # use the file name to name the file
-            dest_path = repo_dir.joinpath(*name_parts[:-1]) / src_path.name
+        rel_path = module_file.relative_to(project_root)
+        dest_path = repo_dir / rel_path
         dest_path.parent.mkdir(parents=True, exist_ok=True)
         shutil.copy2(src_path, dest_path)
+
+        # Ensure __init__.py is copied over at every directory level
+        src_init = project_root / rel_path.parent / "__init__.py"
+        dest_init = dest_path.parent / "__init__.py"
+        if src_init.exists() and not dest_init.exists():
+            shutil.copy2(src_init, dest_init)
+
+    for extra_file in get_extra_files():
+        if extra_file.is_dir():
+            shutil.copytree(extra_file, repo_dir / extra_file.relative_to(project_root))
+        else:
+            shutil.copy2(extra_file, repo_dir / extra_file.relative_to(project_root))
+
     return repo_dir
 
 
 def create_agent_repo(repo_path: str, with_code: bool = True) -> Path:
     """
+    Args:
+        repo_path: The path to the repository.
+        with_code: Whether to include the code in the repository.
+        branch: The branch to post it to.
+        tag: The tag to give it.
     Create a temporary directory inside the Modaic cache. Containing everything that will be pushed to the hub. This function adds the following files:
     - All internal modules used to run the agent
     - The pyproject.toml
@@ -261,23 +278,52 @@ def get_ignored_files() -> list[Path]:
     pyproject_path = Path("pyproject.toml")
     doc = tomlk.parse(pyproject_path.read_text(encoding="utf-8"))
 
-    # Safely get [tool.modaic.ignore]
-    ignore_table = (
+    # Safely get [tool.modaic.exclude]
+    files = (
         doc.get("tool", {})  # [tool]
         .get("modaic", {})  # [tool.modaic]
-        .get("ignore")  # [tool.modaic.ignore]
+        .get("exclude", {})  # [tool.modaic.exclude]
+        .get("files", [])  # [tool.modaic.exclude] files = ["file1", "file2"]
     )
 
-    if ignore_table is None or "files" not in ignore_table:
-        return []
+    excluded: list[Path] = []
+    for entry in files:
+        entry = Path(entry)
+        if not entry.is_absolute():
+            entry = project_root / entry
+        if entry.exists():
+            excluded.append(entry)
+    return excluded
 
-    ignored: list[Path] = []
-    for entry in ignore_table["files"]:
-        try:
-            ignored.append((project_root / entry).resolve())
-        except OSError:
-            continue
-    return ignored
+
+def get_extra_files() -> list[Path]:
+    """Return a list of extra files that should be excluded from staging."""
+    project_root = resolve_project_root()
+    pyproject_path = Path("pyproject.toml")
+    doc = tomlk.parse(pyproject_path.read_text(encoding="utf-8"))
+    files = (
+        doc.get("tool", {})  # [tool]
+        .get("modaic", {})  # [tool.modaic]
+        .get("include", {})  # [tool.modaic.include]
+        .get("files", [])  # [tool.modaic.include] files = ["file1", "file2"]
+    )
+    included: list[Path] = []
+    for entry in files:
+        entry = Path(entry)
+        if entry.is_absolute():
+            try:
+                entry = entry.resolve()
+                entry.relative_to(project_root.resolve())
+            except ValueError:
+                warnings.warn(
+                    f"{entry} will not be bundled because it is not inside the current working directory", stacklevel=4
+                )
+        else:
+            entry = project_root / entry
+        if entry.resolve().exists():
+            included.append(entry)
+
+    return included
 
 
 def create_pyproject_toml(repo_dir: Path, package_name: str):
@@ -293,7 +339,7 @@ def create_pyproject_toml(repo_dir: Path, package_name: str):
     if "project" not in doc_old:
         raise KeyError("No [project] table in old TOML")
     doc_new["project"] = doc_old["project"]
-    doc_new["project"]["dependencies"] = get_filtered_dependencies(doc_old["project"]["dependencies"])
+    doc_new["project"]["dependencies"] = get_final_dependencies(doc_old["project"]["dependencies"])
     if "tool" in doc_old and "uv" in doc_old["tool"] and "sources" in doc_old["tool"]["uv"]:
         doc_new["tool"] = {"uv": {"sources": doc_old["tool"]["uv"]["sources"]}}
         warn_if_local(doc_new["tool"]["uv"]["sources"])
@@ -304,29 +350,32 @@ def create_pyproject_toml(repo_dir: Path, package_name: str):
         tomlk.dump(doc_new, fp)
 
 
-def get_filtered_dependencies(dependencies: list[str]) -> list[str]:
+def get_final_dependencies(dependencies: list[str]) -> list[str]:
     """
     Get the dependencies that should be included in the bundled agent.
+    Filters out "[tool.modaic.ignore] dependencies. Adds [tool.modaic.include] dependencies.
     """
     pyproject_path = Path("pyproject.toml")
     doc = tomlk.parse(pyproject_path.read_text(encoding="utf-8"))
 
-    # Safely get [tool.modaic.ignore]
-    ignore_table = (
+    # Safely get [tool.modaic.exclude]
+    exclude_deps = (
         doc.get("tool", {})  # [tool]
         .get("modaic", {})  # [tool.modaic]
-        .get("ignore", {})  # [tool.modaic.ignore]
+        .get("exclude", {})  # [tool.modaic.exclude]
+        .get("dependencies", [])  # [tool.modaic.exclude] dependencies = ["praw", "sagemaker"]
+    )
+    include_deps = (
+        doc.get("tool", {})  # [tool]
+        .get("modaic", {})  # [tool.modaic]
+        .get("include", {})  # [tool.modaic.include]
+        .get("dependencies", [])  # [tool.modaic.include] dependencies = ["praw", "sagemaker"]
     )
 
-    if "dependencies" not in ignore_table:
-        return dependencies
-
-    ignored_dependencies = ignore_table["dependencies"]
-    if not ignored_dependencies:
-        return dependencies
-    pattern = re.compile(r"\b(" + "|".join(map(re.escape, ignored_dependencies)) + r")\b")
-    filtered_dependencies = [pkg for pkg in dependencies if not pattern.search(pkg)]
-    return filtered_dependencies
+    if exclude_deps:
+        pattern = re.compile(r"\b(" + "|".join(map(re.escape, exclude_deps)) + r")\b")
+        dependencies = [pkg for pkg in dependencies if not pattern.search(pkg)]
+    return dependencies + include_deps
 
 
 def warn_if_local(sources: dict[str, dict]):
@@ -339,3 +388,27 @@ def warn_if_local(sources: dict[str, dict]):
                 f"Bundling agent with local package {source} installed from {config['path']}. This is not recommended.",
                 stacklevel=5,
             )
+
+
+def _module_path(instance: object) -> str:
+    """
+    Return a deterministic module path for the given instance.
+
+    Args:
+      instance: The object instance whose class path should be resolved.
+
+    Returns:
+      str: A fully qualified path in the form "<module>.<ClassName>". If the
+      class' module is "__main__", use the file system to derive a stable
+      module name: the parent directory name when the file is "__main__.py",
+      otherwise the file stem.
+    """
+
+    cls = type(instance)
+    module_name = cls.__module__
+    module = sys.modules[module_name]
+    file = Path(module.__file__)
+    module_path = str(file.relative_to(resolve_project_root()).with_suffix(""))
+    module_path = module_path.replace("/", ".")
+
+    return f"{module_path}.{cls.__name__}"

modaic/observability.py

@@ -38,7 +38,7 @@ _configured = False
 
 
 def configure(
-    tracing: bool = True,
+    tracing: bool = False,
     repo: Optional[str] = None,
     project: Optional[str] = None,
     base_url: str = "https://api.modaic.dev",
@@ -92,7 +92,7 @@ def configure(
         if project_name:
             _opik_client = Opik(host=base_url, project_name=project_name)
 
-        config.update_session_config("track_disable", not tracing)
+    config.update_session_config("track_disable", not tracing)
 
     _configured = True
 
@@ -207,10 +207,17 @@ class Trackable:
     All Modaic classes except PrecompiledAgent should inherit from this class.
     """
 
-    def __init__(self, repo: Optional[str] = None, project: Optional[str] = None, commit: Optional[str] = None):
+    def __init__(
+        self,
+        repo: Optional[str] = None,
+        project: Optional[str] = None,
+        commit: Optional[str] = None,
+        trace: bool = False,
+    ):
         self.repo = repo
         self.project = project
         self.commit = commit
+        self.trace = trace
 
     def set_repo_project(self, repo: Optional[str] = None, project: Optional[str] = None, trace: bool = True):
         """Update the repo and project for this trackable object."""
@@ -218,10 +225,7 @@ class Trackable:
             self.repo = repo
 
         self.project = f"{self.repo}-{project}" if project else self.repo
-
-        # configure global tracing
-        if trace and (repo or project):
-            configure(tracing=trace, repo=repo or self.repo, project=project or self.project)
+        self.trace = trace
 
 
 MethodDecorator = Callable[
@@ -254,8 +258,8 @@ def track_modaic_obj(func: Callable[Concatenate[T, P], R]) -> Callable[Concatena
         repo = getattr(self, "repo", None)
         project = getattr(self, "project", None)
 
-        # check if tracking is enabled globally
-        if not _settings.tracing:
+        # check if tracking is enabled both globally and for this object
+        if not _settings.tracing or not self.trace:
             # binds the method to self so it can be called with args and kwars, also type cast's it to callable with type vars for static type checking
             bound = cast(Callable[P, R], func.__get__(self, type(self)))
             return bound(*args, **kwargs)

modaic/precompiled.py

@@ -4,7 +4,15 @@ import os
 import pathlib
 from abc import ABC, abstractmethod
 from pathlib import Path
-from typing import TYPE_CHECKING, ClassVar, Dict, Generic, List, Optional, Type, TypeVar, Union
+from typing import (
+    TYPE_CHECKING,
+    Dict,
+    List,
+    Optional,
+    Type,
+    TypeVar,
+    Union,
+)
 
 import dspy
 from pydantic import BaseModel
@@ -13,6 +21,7 @@ from modaic.module_utils import create_agent_repo
 from modaic.observability import Trackable, track_modaic_obj
 
 from .hub import load_repo, push_folder_to_hub
+from .module_utils import _module_path
 
 if TYPE_CHECKING:
     from modaic.context.base import Context
@@ -119,6 +128,7 @@ class PrecompiledConfig(BaseModel):
         return self.model_dump_json()
 
 
+# Use a metaclass to enforce super().__init__() with config
 class PrecompiledAgent(dspy.Module):
     """
     Bases: `dspy.Module`
@@ -140,6 +150,7 @@ class PrecompiledAgent(dspy.Module):
     ):
         # create DSPy callback for observability if tracing is enabled
         callbacks = []
+        # FIXME This logic is not correct.
         if trace and (repo or project):
             try:
                 from opik.integrations.dspy.callback import OpikCallback
@@ -150,7 +161,7 @@ class PrecompiledAgent(dspy.Module):
                 elif repo and not project:
                     project_name = repo
                 else:
-                    raise ValueError("Must provide either repo to enable tracing")
+                    raise ValueError("Must provide either repo to enable observability tracking")
 
                 opik_callback = OpikCallback(project_name=project_name, log_graph=True)
                 callbacks.append(opik_callback)
@@ -160,6 +171,7 @@ class PrecompiledAgent(dspy.Module):
 
         # initialize DSPy Module with callbacks
         super().__init__()
+        # FIXME this adds the same callback for every agent. Should only be the current agent.
         if callbacks:
             # set callbacks using DSPy's configuration
             import dspy
@@ -269,7 +281,11 @@ class PrecompiledAgent(dspy.Module):
             with_code: Whether to save the code along with the agent.json and config.json.
         """
         _push_to_hub(
-            self, repo_path=repo_path, access_token=access_token, commit_message=commit_message, with_code=with_code
+            self,
+            repo_path=repo_path,
+            access_token=access_token,
+            commit_message=commit_message,
+            with_code=with_code,
         )
 
 
@@ -363,41 +379,6 @@ class Indexer(Retriever):
         pass
 
 
-def _module_path(instance: object) -> str:
-    """
-    Return a deterministic module path for the given instance.
-
-    Args:
-      instance: The object instance whose class path should be resolved.
-
-    Returns:
-      str: A fully qualified path in the form "<module>.<ClassName>". If the
-      class' module is "__main__", use the file system to derive a stable
-      module name: the parent directory name when the file is "__main__.py",
-      otherwise the file stem.
-    """
-
-    cls = type(instance)
-    module_name = getattr(cls, "__module__", "__main__")
-    class_name = getattr(cls, "__name__", "Object")
-    if module_name != "__main__":
-        return f"{module_name}.{class_name}"
-
-    # When executed as a script, classes often report __module__ == "__main__".
-    # Normalize to a deterministic name based on the defining file path.
-    try:
-        file_path = pathlib.Path(inspect.getfile(cls)).resolve()
-    except Exception:
-        # Fallback to a generic name if the file cannot be determined
-        normalized_root = "main"
-    else:
-        if file_path.name == "__main__.py":
-            normalized_root = file_path.parent.name or "main"
-        else:
-            normalized_root = file_path.stem or "main"
-    return f"{normalized_root}.{class_name}"
-
-
 # CAVEAT: PrecompiledConfig does not support push_to_hub() intentionally,
 # this is to avoid confusion when pushing a config to the hub thinking it
 # will update the config.json when in reality it will overwrite the entire
@@ -414,7 +395,12 @@ def _push_to_hub(
     """
     repo_dir = create_agent_repo(repo_path, with_code=with_code)
     self.save_precompiled(repo_dir, _with_auto_classes=with_code)
-    push_folder_to_hub(repo_dir, repo_path=repo_path, access_token=access_token, commit_message=commit_message)
+    push_folder_to_hub(
+        repo_dir,
+        repo_path=repo_path,
+        access_token=access_token,
+        commit_message=commit_message,
+    )
 
 
 def is_local_path(s: str | Path) -> bool:

modaic/query_language.py

@@ -1,7 +1,7 @@
 from types import NoneType
-from typing import Any, Literal, Optional, Type, TypeAlias, Union
+from typing import Optional, Type, TypeAlias, Union
 
-from langchain_core.structured_query import Comparator, Comparison, Operation, Operator, StructuredQuery, Visitor
+from langchain_core.structured_query import Comparator, Comparison, Operation, Operator, Visitor
 
 ValueType: TypeAlias = Union[int, str, float, bool, NoneType, list, "Value"]
 
@@ -32,30 +32,6 @@ allowed_types = {
 }
 
 
-# def _print_return(func):  # noqa: ANN001
-#     def wrapper(*args, **kwargs):
-#         result = func(*args, **kwargs)
-#         if isinstance(op := args[1], str) and op[0] == "$":
-#             if kwargs.get("recursed", False):
-#                 print(  # noqa: T201
-#                     f"{repr(args[0])} ({mql_operator_to_python[op]}) {repr(args[2])} ->:",
-#                     result,
-#                 )
-#             else:
-#                 print(  # noqa: T201
-#                     f"{repr(args[0])} {mql_operator_to_python[op]} {repr(args[2])} ->:",
-#                     result,
-#                 )
-#         else:
-#             if func.__name__ == "__and__":
-#                 print(f"{repr(args[0])} & {repr(args[1])} ->:", result)  # noqa: T201
-#             elif func.__name__ == "__rand__":
-#                 print(f"{repr(args[1])} & {repr(args[0])} ->:", result)  # noqa: T201
-#         return result
-
-#     return wrapper
-
-
 class Condition:
     """
     Modaic Query Language Property class.

modaic/types.py

@@ -207,6 +207,8 @@ class InnerField(BaseModel):
         # NOTE: handle case where the float type was used and therefore not annotated with a format
         if is_optional:
             raise SchemaError("Array/List elements cannot be None/null")
+        if inspected_type["type"] == "object" or inspected_type["type"] == "array":
+            raise SchemaError("Arrays and Dicts are not supported for Array/List elements")
         return InnerField(
             type=inspected_type["type"],
             format=inspected_type.get("format", None),

modaic/utils.py

@@ -2,9 +2,10 @@ import os
 import re
 from pathlib import Path
 
-from dotenv import load_dotenv
+from dotenv import find_dotenv, load_dotenv
 
-load_dotenv()
+env_file = find_dotenv(usecwd=True)
+load_dotenv(env_file)
 
 
 def compute_cache_dir() -> Path:
@@ -18,4 +19,6 @@ def compute_cache_dir() -> Path:
 
 def validate_project_name(text: str) -> bool:
     """Letters, numbers, underscore, hyphen"""
-    assert bool(re.match(r'^[a-zA-Z0-9_]+$', text)), "Invalid project name. Must contain only letters, numbers, and underscore."
+    assert bool(re.match(r"^[a-zA-Z0-9_]+$", text)), (
+        "Invalid project name. Must contain only letters, numbers, and underscore."
+    )

{modaic-0.1.2.dist-info → modaic-0.3.0.dist-info}/METADATA

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: modaic
-Version: 0.1.2
-Summary: Modular Agent Infrastructure Collective, a python framework for managing and sharing DSPy agents
+Version: 0.3.0
+Summary: Modular Agent Infrastructure Collection, a python framework for managing and sharing DSPy agents
 Author-email: Tyrin <tytodd@mit.edu>, Farouk <farouk@modaic.dev>
 License: MIT License
         
@@ -48,6 +48,7 @@ Requires-Dist: langchain-community>=0.3.29
 Requires-Dist: langchain-core>=0.3.72
 Requires-Dist: langchain-text-splitters>=0.3.9
 Requires-Dist: more-itertools>=10.8.0
+Requires-Dist: openpyxl>=3.1.5
 Requires-Dist: opik==1.8.42
 Requires-Dist: pillow>=11.3.0
 Requires-Dist: pymilvus>=2.5.14
@@ -58,8 +59,11 @@ Requires-Dist: pinecone>=7.3.0; extra == "pinecone"
 Dynamic: license-file
 
 [![Docs](https://img.shields.io/badge/docs-available-brightgreen.svg)](https://docs.modaic.dev)
+[![PyPI](https://img.shields.io/pypi/v/modaic)](https://pypi.org/project/modaic/)
+
+
 # Modaic 🐙
-**Mod**ular **A**gent **I**nfrastructure **C**ollective, a Python framework for building AI agents with structured context management, database integration, and retrieval-augmented generation (RAG) capabilities.
+**Mod**ular **A**gent **I**nfrastructure **C**ollection, a Python framework for building AI agents with structured context management, database integration, and retrieval-augmented generation (RAG) capabilities.
 
 ## Overview
 
@@ -255,7 +259,6 @@ from modaic.databases import VectorDatabase, SQLDatabase
 from modaic.types import Indexer
 
 class TableRAGConfig(PrecompiledConfig):
-    agent_type = "TableRAGAgent"
     k_recall: int = 50
     k_rerank: int = 5
 

{modaic-0.1.2.dist-info → modaic-0.3.0.dist-info}/RECORD

@@ -1,23 +1,24 @@
-modaic/__init__.py,sha256=rf2O0S7OAz5fBkmOcwygsM3u8Nmq_ios01ToNweNnSk,639
-modaic/auto_agent.py,sha256=jegs8HMbE5OICZIDtBrP4kIma_R7rzy-1Kgmb_-eCck,8695
+modaic/__init__.py,sha256=xHu2SUk3OMvb8PIzrVCRS1pBk-Ho9BhwmzKOf_bOjGc,809
+modaic/auto.py,sha256=rPOdQ7s-YGBQLa_v6lVONH8pbOrarPwp4VzRErp0y5c,9091
+modaic/datasets.py,sha256=K-PpPSYIxJI0-yH-SBVpk_EfCM9i_uPz-brmlzP7hzI,513
 modaic/exceptions.py,sha256=XxzxOWjZTzT3l1BqTr7coJnVGxJq53uppRNrqP__YGo,651
-modaic/hub.py,sha256=iWvWjaZxurd2BRXTS2gjhBvqVE1TDtwF0N7Falwj04Q,10527
-modaic/indexing.py,sha256=L0O5yV7AhDUJ0gMyGE17BvHN2gwHxwkOMaxNTkyWQ8g,4185
-modaic/module_utils.py,sha256=DDXUmcGFdaah_EhTlfdHC26ohmMwLNlIYo6PzYzKzqc,10728
-modaic/observability.py,sha256=QEjLbmsVQzWZuxKQU8TBMSHsHVGvTzOeoNlEn_srmyg,9955
-modaic/precompiled.py,sha256=g0AsFrHxzTlDETDqxZFqjHaLqW6m5OpGGyI01xDBT5U,15788
-modaic/query_language.py,sha256=c-La7jYhHgNyjlQaxz0ALvUoiCDGzH9DpSD_9cozxNQ,10463
-modaic/types.py,sha256=sHJ7J9YGfWIkDPfCuRc-n4O9n7g7LNnKSZperAviRFc,9905
-modaic/utils.py,sha256=zbeMlrP_hoo8JUKR_bcuYPxrlYcckY3p0KJ_h4CN5VQ,721
+modaic/hub.py,sha256=d5HQjaE26K1qNCBc32qJtrpESyRv6OiniAteasiN_rk,11290
+modaic/indexing.py,sha256=VdILiXiLVzgV1pSTV8Ho7x1dZtd31Y9z60d_Qtqr2NU,4195
+modaic/module_utils.py,sha256=I6kGCmGSuHM9lxz8rpCIkdFHCh0M7OIyw3GZ3966YWY,13442
+modaic/observability.py,sha256=LgR4gJM4DhD-xlVX52mzRQSPgLQzbeh2LYPmQVqSh-A,9947
+modaic/precompiled.py,sha256=_FwosgvItwnliVOc6nUp15RaBinmo1tTYhR9haun020,14864
+modaic/query_language.py,sha256=BJIigR0HLapiIn9fF7jM7PkLM8OWUDjwYuxmzcCVvyo,9487
+modaic/types.py,sha256=gcx8J4oxrHwxA7McyYV4OKHsuPhhmowJtJIgjJQbLto,10081
+modaic/utils.py,sha256=doJs-XL4TswSQFBINZeKrik-cvjZk-tS9XmWH8fOYiw,794
 modaic/agents/rag_agent.py,sha256=f8s3EILOPUxMpOKDoAvk-cfLE8S9kFNvkEcAC5z2EmQ,798
 modaic/agents/registry.py,sha256=z6GuPxGrq2dinCamiMJ_HVPsD9Tp9XWDUSMZ-uhWPrU,2446
-modaic/context/__init__.py,sha256=WTBo-WqhgeR84P3MEq0snLZyIHnLyv9VYO5Wfp4vrZo,534
-modaic/context/base.py,sha256=QrRNBs05fb5EXN4intHT4D-XIG76RzvrO6j8RXJpzd4,40380
+modaic/context/__init__.py,sha256=FK-bxSu36yGFF1rATy4Yzl4Fpv9kYOlRpBRfr_4moiM,560
+modaic/context/base.py,sha256=x66_lcdQ063DiluC6UnFEH4etgJkbAyukrgrp2KLV5U,40368
 modaic/context/dtype_mapping.py,sha256=xRasW-H92YEuOfH8SbsVnodM9F-90pazott8qF2GWHw,519
-modaic/context/table.py,sha256=c0OUVwglaDRtJ2Uo9z7bAKhIJpJXqwWk1VkKwQNOvUY,17696
-modaic/context/text.py,sha256=S8E-dbTe-Wip8KCZ5vcIOZVyiVQReIL4bSo0anQw828,2581
+modaic/context/table.py,sha256=9Lh2_UyK3OsWmgUfZQa8jDeVAPAk_iueT89_cruDeSo,17699
+modaic/context/text.py,sha256=gCVQx15FrPcmpr2GYkJca6noh7fw_nTcCt-hISwcnvQ,2581
 modaic/databases/__init__.py,sha256=-w_yiY-Sqi1SgcPD5oAQL7MU4VXTihPa1GYGlrHfsFw,784
-modaic/databases/graph_database.py,sha256=vMCYQrnBu5AIIGF_akkU9lWKFdbIDF3tlcyxiNQ_vSQ,9933
+modaic/databases/graph_database.py,sha256=j44PgWGMeD3dtPZe5sRpKnruTA3snm_TiXncusTzqIQ,9990
 modaic/databases/sql_database.py,sha256=wqy7AqsalhmYsbNPy0FCAg1FrUKN6Bd8ytwyJireC94,12057
 modaic/databases/vector_database/__init__.py,sha256=sN1SuSAMC9NHJDOa80BN_olccaHgmiW2Ek57hBvdZWo,306
 modaic/databases/vector_database/vector_database.py,sha256=RsuRemgFV06opY26CekqLLRoAEFYOGl_CMuFETrYS0c,25238
@@ -32,8 +33,8 @@ modaic/databases/vector_database/vendors/qdrant.py,sha256=AbpHGcgLb-kRsJGnwFEktk
 modaic/storage/__init__.py,sha256=Zs-Y_9jfYUE8XVp8z-El0ZXFM_ZVMqM9aQ6fgGPZsf8,131
 modaic/storage/file_store.py,sha256=kSS7gTP_-16wR3Xgq3frF1BZ8Dw8N--kG4V9rrCXPcc,7315
 modaic/storage/pickle_store.py,sha256=fu9jkmmKNE852Y4R1NhOFePLfd2gskhHSXxuq1G1S3I,778
-modaic-0.1.2.dist-info/licenses/LICENSE,sha256=7LMx9j453Vz1DoQbFot8Uhp9SExF5wlOx7c8vw2qhsE,1333
-modaic-0.1.2.dist-info/METADATA,sha256=uwXUFpL8YCXXPaeDBVb5zc-luhpDvysCoyHP4VPsC2s,8569
-modaic-0.1.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-modaic-0.1.2.dist-info/top_level.txt,sha256=RXWGuF-TsW8-17DveTJMPRiAgg_Rf2mq5F3R7tNu6t8,7
-modaic-0.1.2.dist-info/RECORD,,
+modaic-0.3.0.dist-info/licenses/LICENSE,sha256=7LMx9j453Vz1DoQbFot8Uhp9SExF5wlOx7c8vw2qhsE,1333
+modaic-0.3.0.dist-info/METADATA,sha256=Cw0LkkLDfVGdxif1ZSx9HI9q9c5HSIuzDI-VVWRFRO4,8651
+modaic-0.3.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+modaic-0.3.0.dist-info/top_level.txt,sha256=RXWGuF-TsW8-17DveTJMPRiAgg_Rf2mq5F3R7tNu6t8,7
+modaic-0.3.0.dist-info/RECORD,,