modaic 0.1.0__py3-none-any.whl

modaic/query_language.py

@@ -0,0 +1,321 @@
+from types import NoneType
+from typing import Any, Literal, Optional, Type, TypeAlias, Union
+
+from langchain_core.structured_query import Comparator, Comparison, Operation, Operator, StructuredQuery, Visitor
+
+ValueType: TypeAlias = Union[int, str, float, bool, NoneType, list, "Value"]
+
+value_types = (int, str, float, bool, NoneType, list)
+
+comparator_to_python = {
+    Comparator.EQ: "==",
+    Comparator.LT: "<",
+    Comparator.LTE: "<=",
+    Comparator.GT: ">",
+    Comparator.GTE: ">=",
+    Comparator.NE: "!=",
+    Comparator.CONTAIN: "contains",
+    Comparator.IN: "in_",
+    Comparator.NIN: "not_in",
+}
+
+allowed_types = {
+    Comparator.EQ: [int, str, list, dict, bool, NoneType],
+    Comparator.LT: [int, float],
+    Comparator.LTE: [int, float],
+    Comparator.GT: [int, float],
+    Comparator.GTE: [int, float],
+    Comparator.NE: [int, str, list, dict, bool, NoneType],
+    Comparator.CONTAIN: [int, str, bool, NoneType],
+    Comparator.IN: [list, dict],
+    Comparator.NIN: [list, dict],
+}
+
+
+# def _print_return(func):  # noqa: ANN001
+#     def wrapper(*args, **kwargs):
+#         result = func(*args, **kwargs)
+#         if isinstance(op := args[1], str) and op[0] == "$":
+#             if kwargs.get("recursed", False):
+#                 print(  # noqa: T201
+#                     f"{repr(args[0])} ({mql_operator_to_python[op]}) {repr(args[2])} ->:",
+#                     result,
+#                 )
+#             else:
+#                 print(  # noqa: T201
+#                     f"{repr(args[0])} {mql_operator_to_python[op]} {repr(args[2])} ->:",
+#                     result,
+#                 )
+#         else:
+#             if func.__name__ == "__and__":
+#                 print(f"{repr(args[0])} & {repr(args[1])} ->:", result)  # noqa: T201
+#             elif func.__name__ == "__rand__":
+#                 print(f"{repr(args[1])} & {repr(args[0])} ->:", result)  # noqa: T201
+#         return result
+
+#     return wrapper
+
+
+class Condition:
+    """
+    Modaic Query Language Property class.
+    """
+
+    is_operation: bool = False
+    condition: Comparison | Operation = None
+
+    def __init__(self, condition: Comparison | Operation):
+        if isinstance(condition, Comparison):
+            self.is_operation = False
+        else:
+            self.is_operation = True
+        self.condition = condition
+
+    def __repr__(self):
+        return f"Condition({repr(self.condition)})"
+
+    def __str__(self):
+        return (
+            f"Prop({self.condition.attribute}) {comparator_to_python[self.condition.comparator]} {self.condition.value}"
+        )
+
+    def __contains__(self, other: str):
+        raise ValueError("Modaic Filters do not support `in` use Prop.in_()/Prop.not_in() instead")
+
+    def __bool__(self):
+        raise ValueError(
+            "Attempted to evaluate Modaic condition as boolean. Please make sure you wrap ALL expresions with ()"
+        )
+
+    def __and__(self, other: Union["Condition", ValueType]):
+        if not isinstance(other, Condition):
+            other = Value(other)
+        return AND(self, other)
+
+    # @_print_return
+    def __or__(self, other: Union["Condition", ValueType]):
+        if not isinstance(other, Condition):
+            other = Value(other)
+        return OR(self, other)
+
+    # @_print_return
+    def __rand__(self, other: Union[int, str, list]):
+        if not isinstance(other, Condition):
+            other = Value(other)
+        return AND(other, self)
+
+    # @_print_return
+    def __ror__(self, other: Union[int, str, list]):
+        if not isinstance(other, Condition):
+            other = Value(other)
+        return OR(other, self)
+
+    def __invert__(self):
+        # TODO: implement , use nor
+        raise NotImplementedError("Modaic queires do not support ! yet")
+
+
+def _enforce_types(
+    other: ValueType,
+    op: str,
+    allowed_types: list[Type],
+):
+    other_type = _get_type(other)
+    if other_type not in allowed_types:
+        raise ValueError(f"Value must be one of {allowed_types}, got {other_type} for {op}")
+
+
+def _get_type(value: ValueType):
+    if isinstance(value, Value):
+        return type(value.value)
+    elif isinstance(value, value_types):
+        return type(value)
+    elif isinstance(value, Condition):
+        return bool
+    elif isinstance(value, Prop):
+        raise NotImplementedError("Modaic queries do not support comparisions between properties yet.")
+    else:
+        raise ValueError(f"Unssuported value type in Modaic condition, got {type(value)}")
+
+
+class Prop:
+    """
+    Modaic Query Language Property class.
+    """
+
+    def __init__(
+        self,
+        name: str,
+    ):
+        super().__init__()
+        self.name = name
+
+    def __getitem__(self, key: str):
+        raise NotImplementedError("Modaic queries do not support nested properties yet")
+
+    def in_(self, other: Union["Value", list]) -> "Condition":
+        value = Value(other)
+        return self.comparison("in", Comparator.IN, value)
+
+    def not_in(self, other: Union["Value", list]) -> "Condition":
+        value = Value(other)
+        return self.comparison("not_in", Comparator.NIN, value)
+
+    def __eq__(self, other: Optional[Union[ValueType]]) -> "Condition":
+        value = Value(other)
+        return self.comparison("==", Comparator.EQ, value)
+
+    def __lt__(self, other: Union[ValueType]) -> "Condition":
+        value = Value(other)
+        return self.comparison("<", Comparator.LT, value)
+
+    def __le__(self, other: Union[ValueType]) -> "Condition":
+        value = Value(other)
+        return self.comparison("<=", Comparator.LTE, value)
+
+    def __gt__(self, other: Union[ValueType]) -> "Condition":
+        value = Value(other)
+        return self.comparison(">", Comparator.GT, value)
+
+    def __ge__(self, other: Union[ValueType]) -> "Condition":
+        value = Value(other)
+        return self.comparison(">=", Comparator.GTE, value)
+
+    def __ne__(self, other: Optional[Union[ValueType]]) -> "Condition":
+        value = Value(other)
+        return self.comparison("!=", Comparator.NE, value)
+
+    def contains(self, other: Union[ValueType]) -> "Condition":
+        value = Value(other)
+        return self.comparison("contains", Comparator.CONTAIN, value)
+
+    def exists(self) -> "Condition":
+        # TODO: implement
+        raise NotImplementedError("Prop does not support exists")
+
+    def not_exists(self) -> "Condition":
+        # TODO: implement
+        raise NotImplementedError("Prop does not support not_exists")
+
+    def comparison(self, op_expr: str, comparator: Comparator, other: "Value") -> "Condition":
+        _enforce_types(other, op_expr, allowed_types[comparator])
+        comparison = Comparison(
+            comparator=comparator,
+            attribute=self.name,
+            value=other.value,
+        )
+        return Condition(comparison)
+
+    def __len__(self):
+        # TODO: implement
+        raise NotImplementedError("Prop does not support __len__")
+
+    def all(self, other):  # noqa: ANN001
+        # TODO: implement
+        raise NotImplementedError("Prop does not support all")
+
+    def any(self, other):  # noqa: ANN001
+        # TODO: implement
+        raise NotImplementedError("Prop does not support any")
+
+
+class Value:
+    """
+    Modaic Query Language Value class.
+    """
+
+    def __init__(self, value: Union[int, str, list, dict, bool, "Value", None]):
+        super().__init__()
+        if not isinstance(value, ValueType):
+            raise ValueError(
+                f"Value must be one of {value_types}, got {type(value)}. Please wrap your expressions with ()"
+            )
+        if isinstance(value, Value):
+            value = value.value
+        self.value = value
+
+
+class AND(Condition):
+    """
+    Modaic Query Language AND class.
+    """
+
+    is_operation: bool = True
+    condition: Operation
+
+    def __init__(self, left: "Condition", right: "Condition"):
+        self.left = left
+        self.right = right
+        if isinstance(self.left, AND) and isinstance(self.right, AND):
+            arguments = self.left.condition.arguments + self.right.condition.arguments
+        elif and_other := _get_and_other(self.left, self.right):
+            arguments = and_other[0].condition.arguments + [and_other[1].condition]
+        else:
+            arguments = [self.left.condition, self.right.condition]
+        super().__init__(Operation(operator=Operator.AND, arguments=arguments))
+
+    def __repr__(self):
+        return f"AND({self.left}, {self.right})"
+
+    def __str__(self):
+        return f"({str(self.left)}) & ({str(self.right)})"
+
+
+class OR(Condition):
+    """
+    Modaic Query Language OR class.
+    """
+
+    is_operation: bool = True
+    condition: Operation
+
+    def __init__(self, left: "Condition", right: "Condition", complete: bool = False):
+        self.left = left
+        self.right = right
+        if isinstance(self.left, OR) and isinstance(self.right, OR):
+            arguments = self.left.condition.arguments + self.right.condition.arguments
+        elif or_other := _get_or_other(self.left, self.right):
+            arguments = or_other[0].condition.arguments + [or_other[1].condition]
+        else:
+            arguments = [self.left.condition, self.right.condition]
+        super().__init__(Operation(operator=Operator.OR, arguments=arguments))
+
+    def __repr__(self):
+        return f"OR({self.left}, {self.right})"
+
+    def __str__(self):
+        return f"({str(self.left)}) | ({str(self.right)})"
+
+
+def _get_and_or(left: "Condition", right: "Condition"):
+    if isinstance(left, AND) and isinstance(right, OR):
+        return left, right
+    elif isinstance(right, AND) and isinstance(left, OR):
+        return right, left
+    else:
+        return None
+
+
+def _get_and_other(left: "Condition", right: "Condition"):
+    if isinstance(left, AND) and type(right) is Condition:
+        return left, right
+    elif isinstance(right, AND) and type(left) is Condition:
+        return right, left
+    else:
+        return None
+
+
+def _get_or_other(left: "Condition", right: "Condition"):
+    if isinstance(left, OR) and right is Condition:
+        return left, right
+    elif isinstance(right, OR) and left is Condition:
+        return right, left
+    else:
+        return None
+
+
+def parse_modaic_filter(translator: Visitor, condition: Condition) -> dict:  # noqa: N802
+    if condition.is_operation:
+        return translator.visit_operation(condition.condition)
+    else:
+        return translator.visit_comparison(condition.condition)

modaic/storage/__init__.py

@@ -0,0 +1,3 @@
+from .file_store import FileStore, InPlaceFileStore, LocalFileStore
+
+__all__ = ["FileStore", "InPlaceFileStore", "LocalFileStore"]

modaic/storage/file_store.py

@@ -0,0 +1,239 @@
+import shutil
+import uuid
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import (
+    IO,
+    Iterator,
+    NamedTuple,
+    Optional,
+    Tuple,
+)
+
+import immutables
+
+
+class FileResult(NamedTuple):
+    """
+    Return type for FileStore.get()
+    Args:
+        file: The file object
+        type: The type of the file without the leading dot (e.g. "csv", "txt", "xlsx", etc.). Usually the extension of the file but can be used for other purposes. (e.g. "google_doc")
+        name: The name of the file
+        metadata: A map of metadata for the file
+    """
+
+    file: Path | IO
+    type: str
+    name: Optional[str] = None
+    metadata: immutables.Map = immutables.Map()
+
+
+class FileStore(ABC):
+    """
+    Base class for file stores. File stores are used to store files that hydrate Context objects. The files can be stored locally or remotely.
+    """
+
+    @abstractmethod
+    def get(self, reference: str) -> FileResult:
+        """
+        Get a file from the file store by reference (like an id or path).
+
+        Args:
+            reference: The id or path of the file in the FileStore.
+
+        Raises:
+            FileNotFoundError: If the file is not found.
+
+        Returns:
+            A Path object or a file-like object.
+        """
+
+    @abstractmethod
+    def contains(self, reference: str) -> bool:
+        """
+        Checks if the file store contains a given reference.
+
+        Args:
+            reference: The id or path of the file in the FileStore.
+
+        Returns:
+            True if the reference is found in the file store, False otherwise.
+        """
+        pass
+
+    @abstractmethod
+    def keys(self, folder: Optional[str] = None) -> Iterator[str]:
+        """
+        Iterate over all keys in the file store.
+        """
+        pass
+
+    def values(self, folder: Optional[str] = None) -> Iterator[FileResult]:
+        """
+        Iterate over all files in the file store.
+        """
+        for ref in self.keys(folder):
+            yield self.get(ref)
+
+    def items(self, folder: Optional[str] = None) -> Iterator[Tuple[str, FileResult]]:
+        """
+        Iterate over all keys and files in the file store.
+        """
+        for ref in self.keys(folder):
+            yield ref, self.get(ref)
+
+    def __iter__(self) -> Iterator[str]:
+        """
+        Iterate over all references in the file store.
+        """
+        return self.keys()
+
+    def __contains__(self, reference: str) -> bool:
+        return self.contains(reference)
+
+    def __len__(self) -> int:
+        """
+        Get the number of files in the file store.
+        """
+        return len(list(self.keys()))
+
+
+class MutableFileStore(FileStore):
+    """
+    A FileStore that supports adding and removing files.
+    """
+
+    @abstractmethod
+    def add(self, file: str | Path | IO, type: Optional[str] = None) -> str:
+        """
+        Add a file to the file store.
+
+        Args:
+            file: The file to add to the file store.
+
+        Returns:
+            The reference to the file in the FileStore.
+        """
+
+    @abstractmethod
+    def update(self, reference: str, file: str | Path | IO, type: Optional[str] = None) -> None:
+        """
+        Update a file in the file store.
+
+        Args:
+            reference: The id or path of the file in the FileStore.
+            file: The file to update in the file store.
+
+        Raises:
+            FileNotFoundError: If the file is not found.
+        """
+        pass
+
+    @abstractmethod
+    def remove(self, reference: str) -> None:
+        """
+        Remove a file from the file store.
+
+        Raises:
+            FileNotFoundError: If the file is not found.
+
+        Args:
+            reference: The id or path of the file in the FileStore.
+        """
+
+
+class InPlaceFileStore(FileStore):
+    def __init__(
+        self,
+        directory: str | Path,
+    ):
+        self.directory = Path(directory)
+        if not (self.directory).exists():
+            raise FileNotFoundError(f"File store directory {self.directory} is not a valid directory")
+
+    def get(self, reference: str) -> FileResult:
+        """
+        Get a file from the file store by path relative to self.directory.
+
+        Args:
+            reference: The local path of the file relative to self.directory.
+
+        Raises:
+            FileNotFoundError: If the file is not found.
+
+        Returns:
+            A Path object.
+
+        """
+        path = self.directory / reference
+        if not path.exists():
+            raise FileNotFoundError(f"File {reference} not found in {self.directory}")
+        return FileResult(file=path, type=path.suffix.lstrip("."), name=path.name)
+
+    def contains(self, reference: str) -> bool:
+        return (self.directory / reference).exists()
+
+    def keys(self, folder: Optional[str] = None) -> Iterator[str]:
+        """
+        Iterate over all files in the file store.
+        """
+        folder: Path = self.directory if folder is None else self.directory / folder
+        return (str(path.relative_to(self.directory)) for path in folder.iterdir() if path.is_file())
+
+
+class LocalFileStore(MutableFileStore):
+    def __init__(self, directory: str | Path):
+        self.directory = Path(directory)
+        (self.directory / ".modaic").mkdir(parents=True, exist_ok=True)
+
+    def get(self, reference: str) -> FileResult:
+        """
+        Get a file from the file store by path relative to self.directory.
+        """
+        path = self.directory / reference
+        if not path.exists():
+            raise FileNotFoundError(f"File {reference} not found in {self.directory}")
+        return FileResult(file=path, type=path.suffix.lstrip("."), name=path.name)
+
+    def contains(self, reference: str) -> bool:
+        return (self.directory / reference).exists()
+
+    def add(self, file: str | Path | IO) -> str:
+        default_filename = str(uuid.uuid4())
+        if isinstance(file, (Path, str)):
+            original_path = Path(file)
+            if is_in_dir(original_path, self.directory):
+                path = original_path
+            else:
+                path = self.directory / ".modaic" / default_filename
+                shutil.copy2(original_path, path)
+        else:
+            path = self.directory / ".modaic" / default_filename
+            with open(path, "wb") as f:
+                shutil.copyfileobj(file, f)
+        return str(path.relative_to(self.directory))
+
+    def update(self, reference: str, file: str | Path | IO) -> None:
+        if isinstance(file, (Path, str)):
+            original_path = Path(file)
+            shutil.copy2(original_path, self.directory / reference)
+        else:
+            shutil.copyfileobj(file, self.directory / reference)
+
+    def remove(self, reference: str) -> None:
+        (self.directory / reference).unlink()
+
+    def keys(self, folder: Optional[str] = None) -> Iterator[str]:
+        base: Path = self.directory if folder is None else self.directory / folder
+        return (str(path.relative_to(self.directory)) for path in base.iterdir() if path.is_file())
+
+
+def is_in_dir(path: str | Path, directory: str | Path) -> bool:
+    path = Path(path).resolve()  # follows symlinks
+    directory = Path(directory).resolve()
+    try:
+        path.relative_to(directory)
+        return True
+    except ValueError:
+        return False

modaic/storage/pickle_store.py

@@ -0,0 +1,25 @@
+import os
+import pickle
+import uuid
+from typing import List
+
+from ..context.base import Context
+
+
+class ContextPickleStore:
+    def __init__(self, directory: str):
+        self.directory = directory
+        os.makedirs(self.directory, exist_ok=True)
+
+    def add(self, contexts: List[Context], **kwargs) -> List[Context]:
+        for context in contexts:
+            context_id = uuid.uuid4()
+            file_name = f"{context_id}.pkl"
+            with open(os.path.join(self.directory, file_name), "wb") as f:
+                pickle.dump(context, f)
+            context.metadata["context_id"] = context_id
+        return contexts
+
+    def get(self, source: str) -> Context:
+        with open(os.path.join(self.directory, source), "rb") as f:
+            return pickle.load(f)