msgflux 0.1.0__py3-none-any.whl

msgflux/auto/params.py

@@ -0,0 +1,167 @@
+"""AutoParams metaclass for automatic parameter management."""
+
+from types import FunctionType, MethodType
+from typing import Any, Dict
+
+
+class AutoParams(type):
+    """Metaclass that captures class attributes and uses them as default parameters.
+
+    This metaclass automatically captures all non-callable, non-special attributes
+    defined in a class and makes them available as default parameter values when
+    instantiating the class.
+
+    The captured parameters are stored in the class's _auto_params attribute and
+    are automatically merged with any parameters passed during instantiation.
+
+    Basic Example:
+        >>> from msgflux.auto import AutoParams
+        >>>
+        >>> class Model(metaclass=AutoParams):
+        ...     def __init__(self, learning_rate, batch_size, epochs):
+        ...         self.learning_rate = learning_rate
+        ...         self.batch_size = batch_size
+        ...         self.epochs = epochs
+        ...
+        >>> class MyModel(Model):
+        ...     learning_rate = 0.001
+        ...     batch_size = 32
+        ...     epochs = 100
+        ...
+        >>> # Uses default values from class attributes
+        >>> m = MyModel()
+        >>> print(m.learning_rate)  # 0.001
+        >>>
+        >>> # Partial override
+        >>> m2 = MyModel(learning_rate=0.01, batch_size=64)
+        >>> print(m2.learning_rate)  # 0.01
+        >>> print(m2.epochs)  # 100 (still uses default)
+
+    Docstring as Parameter:
+        Classes can configure AutoParams to use their docstring as a parameter value
+        by setting the _autoparams_use_docstring_for class attribute.
+
+        >>> class Agent(metaclass=AutoParams):
+        ...     _autoparams_use_docstring_for = "description"
+        ...     def __init__(self, name, description=None):
+        ...         self.name = name
+        ...         self.description = description
+        ...
+        >>> class MyAgent(Agent):
+        ...     '''An agent that helps with coding tasks'''
+        ...
+        >>> agent = MyAgent()
+        >>> print(agent.description)  # "An agent that helps with coding tasks"
+        >>>
+        >>> # Explicit attribute takes precedence over docstring
+        >>> class MyAgent2(Agent):
+        ...     '''This is ignored'''
+        ...     description = "Explicit description"
+        ...
+        >>> agent2 = MyAgent2()
+        >>> print(agent2.description)  # "Explicit description"
+
+    Class Name as Parameter:
+        Classes can configure AutoParams to use the class name as a parameter value
+        by setting the _autoparams_use_classname_for class attribute.
+
+        >>> class Agent(metaclass=AutoParams):
+        ...     _autoparams_use_classname_for = "name"
+        ...     def __init__(self, name):
+        ...         self.name = name
+        ...
+        >>> class SuperAgent(Agent):
+        ...     pass
+        ...
+        >>> agent = SuperAgent()
+        >>> print(agent.name)  # "SuperAgent"
+        >>>
+        >>> # Explicit attribute takes precedence over class name
+        >>> class CustomAgent(Agent):
+        ...     name = "my_custom_agent"
+        ...
+        >>> agent2 = CustomAgent()
+        >>> print(agent2.name)  # "my_custom_agent"
+    """
+
+    def __new__(mcls, name: str, bases: tuple, namespace: Dict[str, Any]):
+        """Create a new class with auto parameter management.
+
+        Args:
+            name: Name of the class being created
+            bases: Base classes
+            namespace: Class namespace dictionary
+
+        Returns:
+            The newly created class with _auto_params attribute
+        """
+        # Collect auto params from base classes
+        inherited_params = {}
+        docstring_param_name = None
+        classname_param_name = None
+        for base in bases:
+            if hasattr(base, "_auto_params"):
+                inherited_params.update(base._auto_params)
+            # Check if base class wants to use docstring as a parameter
+            if hasattr(base, "_autoparams_use_docstring_for"):
+                docstring_param_name = base._autoparams_use_docstring_for
+            # Check if base class wants to use class name as a parameter
+            if hasattr(base, "_autoparams_use_classname_for"):
+                classname_param_name = base._autoparams_use_classname_for
+
+        # Capture all non-callable and non-special attributes from this class
+        # Exclude properties, classmethods, staticmethods, and functions/methods
+        # Allow classes (types) and instances (even if callable) to be collected
+        class_params = {
+            k: v
+            for k, v in namespace.items()
+            if not k.startswith("_")
+            and not isinstance(
+                v, (FunctionType, MethodType, classmethod, staticmethod, property)
+            )
+        }
+
+        # Handle class name as parameter if configured
+        if classname_param_name and classname_param_name not in class_params:
+            # Use the class name being created
+            class_params[classname_param_name] = name
+
+        # Handle docstring as parameter if configured
+        if docstring_param_name and "__doc__" in namespace and namespace["__doc__"]:
+            # Only use docstring if the parameter isn't already defined
+            if docstring_param_name not in class_params:
+                class_params[docstring_param_name] = namespace["__doc__"]
+
+        # Merge inherited params with class params (class params take precedence)
+        all_params = {**inherited_params, **class_params}
+
+        # Store the merged params in the namespace
+        namespace["_auto_params"] = all_params
+
+        return super().__new__(mcls, name, bases, namespace)
+
+    def __call__(cls, *args, **kwargs):
+        """Instantiate the class with auto parameters.
+
+        Merges the class's _auto_params with the provided kwargs, with kwargs
+        taking precedence. This allows for partial or complete override of
+        default parameters.
+
+        Args:
+            *args: Positional arguments to pass to __init__
+            **kwargs: Keyword arguments to pass to __init__
+
+        Returns:
+            Instance of the class
+        """
+        # Start with auto params
+        all_kwargs = dict(cls._auto_params) if hasattr(cls, "_auto_params") else {}
+
+        # Override with user-provided kwargs
+        all_kwargs.update(kwargs)
+
+        # Instantiate normally
+        return super().__call__(*args, **all_kwargs)
+
+
+__all__ = ["AutoParams"]

msgflux/cache.py

@@ -0,0 +1,54 @@
+from functools import wraps
+from typing import TYPE_CHECKING, Optional
+
+from msgflux.core.message import Message
+
+if TYPE_CHECKING:
+    from msgflux.data.dbs.base import BaseDB
+    from msgflux.models.types import TextEmbedderModel
+
+
+def response_cache(
+    db: "BaseDB",
+    task: Optional[str] = None,
+    response_mode: Optional[str] = None,
+    model: Optional["TextEmbedderModel"] = None,
+):
+    def decorator(func):
+        @wraps(func)
+        def wrapper(msg):
+            if isinstance(msg, Message):
+                if task is None:
+                    raise ValueError(
+                        "`task` is required when input is a Message instance"
+                    )
+                key = msg.get(task)
+            else:
+                key = msg
+
+            if model:
+                key = model(key)
+
+            cached_response = db(key)
+
+            if cached_response is not None:
+                if isinstance(msg, Message):
+                    if response_mode is None:
+                        raise ValueError(
+                            "`response_mode` is required when "
+                            "input is a Message instance"
+                        )
+                    msg.set(response_mode, cached_response)
+                    return msg
+                return cached_response
+
+            response = func(msg)
+
+            if response is not None:
+                db.add(key, response)
+
+            return response
+
+        return wrapper
+
+    return decorator

msgflux/core/__init__.py

@@ -0,0 +1,5 @@
+from msgflux.core.dotdict import dotdict
+from msgflux.core.examples import Example, ExampleCollection
+from msgflux.core.message import Message
+
+__all__ = ["Example", "ExampleCollection", "Message", "dotdict"]

msgflux/core/dotdict.py

@@ -0,0 +1,264 @@
+from typing import Any, Iterable, Mapping, Optional, Tuple, Union
+
+import msgspec
+
+
+class dotdict(dict):  # noqa: N801
+    """A dictionary with dot access and nested path support.
+
+    dotdict allows you to access and modify values as attributes (e.g., `obj.key`)
+    and also allows reading and writing nested paths using strings with dot separators
+    (e.g., `obj.get("user.profile.name")`).
+
+    Main features:
+    - Dot access (`obj.key`)
+    - Traditional square bracket access (`obj['key']`)
+    - Nested reading via `.get("a.b.c")`
+    - Nested writing via `.set("a.b.c", value)`
+    - Conversion to standard dict with `.to_dict()`
+    - Support for Msgspec serialization (`__json__`)
+    - Support for lists with path indices (e.g., `"items.0.name"`)
+    - Optional immutability (`frozen=True`)
+    - Hidden keys support (`hidden_keys=["key1", "key2"]`)
+
+    Hidden Keys:
+        The `hidden_keys` parameter marks keys as invisible to enumeration and
+        discovery. Hidden keys are excluded from `keys()`, `values()`, `items()`,
+        `__iter__`, `__contains__` (`in` operator), `to_dict()`, `to_json()`,
+        `__repr__`, and `__str__`.
+
+        Hidden keys are still accessible via direct access: dot notation
+        (`obj.api_key`), bracket notation (`obj['api_key']`), and `get()` paths
+        (`obj.get('api_key')`). This makes them suitable for sensitive data like
+        API keys or passwords that should not leak into logs, serialized output,
+        or iteration, while remaining readable when explicitly requested.
+
+    Example:
+            >>> d = dotdict(
+            ...     {"api_key": "secret", "username": "john"},
+            ...     hidden_keys=["api_key"]
+            ... )
+            >>> "api_key" in d        # False
+            >>> list(d.keys())        # ["username"]
+            >>> d.to_dict()           # {"username": "john"}
+            >>> d.api_key             # "secret"
+            >>> d["api_key"]          # "secret"
+            >>> d.get("api_key")      # "secret"
+    """
+
+    def __init__(
+        self,
+        initial_data: Optional[
+            Union[Mapping[str, Any], Iterable[Tuple[str, Any]]]
+        ] = None,
+        *,
+        frozen: Optional[bool] = False,
+        hidden_keys: Optional[list[str]] = None,
+        **kwargs,
+    ):
+        """Initializes an instance of dotdict.
+
+        Args:
+            initial_data:
+                Base dictionary to initialize data.
+            frozen:
+                If True, prevents changes after creation.
+            hidden_keys:
+                List of keys that should not be returned by get() method.
+                These keys will return None (or default) when accessed via get().
+            **kwargs:
+                Additional key=value pairs.
+
+        ::: example:
+            d = dotdict({"user": {"name": "Maria"}}, frozen=False)
+            print(d.user.name)
+            >> Maria
+
+            # With hidden keys
+            d = dotdict({"api_key": "secret", "name": "John"}, hidden_keys=["api_key"])
+            print(d.get("api_key"))   # None
+            print("api_key" in d)     # False
+            print(d.to_dict())        # {"name": "John"}
+            print(d.api_key)          # "secret"  (direct access)
+        """
+        if initial_data is None:
+            initial_data = {}
+        elif not isinstance(initial_data, dict):
+            initial_data = dict(initial_data)
+        self._frozen = frozen
+        self._hidden_keys = set(hidden_keys or [])
+        super().__init__()
+        for key, value in {**initial_data, **kwargs}.items():
+            super().__setitem__(key, self._wrap(value))
+
+    def __iter__(self):
+        hidden = getattr(self, "_hidden_keys", set())
+        for key in super().__iter__():
+            if key not in hidden:
+                yield key
+
+    def __contains__(self, key):
+        hidden = getattr(self, "_hidden_keys", set())
+        if key in hidden:
+            return False
+        return super().__contains__(key)
+
+    def keys(self):
+        hidden = getattr(self, "_hidden_keys", set())
+        return [k for k in super().keys() if k not in hidden]
+
+    def values(self):
+        hidden = getattr(self, "_hidden_keys", set())
+        return [v for k, v in super().items() if k not in hidden]
+
+    def items(self):
+        hidden = getattr(self, "_hidden_keys", set())
+        return [(k, v) for k, v in super().items() if k not in hidden]
+
+    def __getattr__(self, attr: str):
+        try:
+            return self[attr]
+        except KeyError as e:
+            raise AttributeError(f"`dotdict` object has no attribute '{attr}'") from e
+
+    def __setattr__(self, key: str, value: Any):
+        if key.startswith("_"):
+            super().__setattr__(key, value)
+        elif hasattr(self, "_frozen") and self._frozen:
+            raise AttributeError("Cannot modify frozen dotdict")
+        else:
+            self[key] = value
+
+    def __setitem__(self, key: str, value: Any):
+        if getattr(self, "_frozen", False):
+            raise AttributeError("Cannot modify frozen dotdict")
+        super().__setitem__(key, self._wrap(value))
+
+    def __delattr__(self, key: str):
+        if getattr(self, "_frozen", False):
+            raise AttributeError("Cannot delete from frozen dotdict")
+        try:
+            del self[key]
+        except KeyError as e:
+            raise AttributeError(f"`dotdict` object has no attribute `{key}`") from e
+
+    def _wrap(self, value: Any):
+        if isinstance(value, dict):
+            return dotdict(
+                value,
+                frozen=getattr(self, "_frozen", False),
+                hidden_keys=list(getattr(self, "_hidden_keys", set())),
+            )
+        elif isinstance(value, list):
+            return [self._wrap(item) for item in value]
+        return value
+
+    def get(self, path: str, default: Any = None) -> Any:
+        """Access nested values via dot path.
+
+        !!! example:
+            get('user.profile.age').
+        """
+        keys = path.split(".")
+        current = self
+        try:
+            for key in keys:
+                if isinstance(current, list):
+                    key = int(key)  # noqa: PLW2901
+                current = current[key]
+            return current
+        except (KeyError, IndexError, ValueError, TypeError):
+            return default
+
+    def set(self, path: str, value: Any):
+        """Set nested value via dot path.
+
+        !!! example
+
+            set('user.profile.age', 31).
+        """
+        if self._frozen:
+            raise AttributeError("Cannot modify frozen dotdict")
+
+        keys = path.split(".")
+        current = self
+        for i, key in enumerate(keys):
+            if isinstance(current, list):
+                key = int(key)  # noqa: PLW2901
+
+            if i == len(keys) - 1:
+                if isinstance(current, list):
+                    key_i = int(key)
+                    current[key_i] = self._wrap(value)
+                else:
+                    current[key] = self._wrap(value)
+                return
+
+            if isinstance(current, list):
+                key_i = int(key)
+                if key_i >= len(current) or not isinstance(
+                    current[key_i], (dict, dotdict)
+                ):
+                    current[key_i] = dotdict()
+                current = current[key_i]
+            else:
+                if key not in current or not isinstance(
+                    current[key], (dict, dotdict, list)
+                ):
+                    current[key] = dotdict()
+                current = current[key]
+
+    def update(self, *args, **kwargs):
+        """Extends dict.update to support nested keys while maintaining DotDict."""
+        if self._frozen:
+            raise AttributeError("Cannot modify frozen DotDict")
+
+        # Collects all key-value pairs
+        other = {}
+        if args:
+            if len(args) > 1:
+                raise TypeError(
+                    f"update expected at most 1 arguments, given `{len(args)}`"
+                )
+            other.update(args[0])
+        other.update(kwargs)
+
+        for key, value in other.items():
+            # Nested key with dot?
+            if isinstance(key, str) and "." in key:
+                self.set(key, value)
+
+            # Value is dict and there is already a dotdict on that key?
+            # Merge recursively
+            elif (
+                isinstance(value, dict)
+                and key in self
+                and isinstance(self[key], dotdict)
+            ):
+                self[key].update(value)
+
+            # General case: normal assignment (call __setitem__ e wrap)
+            else:
+                self[key] = value
+
+    def to_dict(self):
+        def unwrap(value):
+            if isinstance(value, dotdict):
+                return {k: unwrap(v) for k, v in value.items()}
+            elif isinstance(value, list):
+                return [unwrap(item) for item in value]
+            return value
+
+        return unwrap(self)
+
+    def to_json(self) -> bytes:
+        """Returns a encoded-JSON."""
+        return msgspec.json.encode(self.to_dict())
+
+    def __repr__(self):
+        d = self.to_dict()
+        attrs_str = "\n".join(f"   '{k}': {v!r}" for k, v in d.items())
+        return f"{self.__class__.__name__}({{\n{attrs_str}\n}})"
+
+    def __str__(self):
+        return str(self.to_dict())

msgflux/core/examples.py

@@ -0,0 +1,169 @@
+from dataclasses import dataclass, field
+from typing import Any, List, Mapping, Optional, Union
+
+from msgflux.utils.xml import apply_xml_tags
+
+
+@dataclass
+class Example:
+    """Represents a single example for a LM.
+
+    Args:
+        inputs:
+            Task inputs.
+        labels:
+            Expected model output.
+        title:
+            Example title.
+        reasoning:
+            Intermediate thinking/logic before generating output.
+        topic:
+            Example topic/category.
+    """
+
+    inputs: Union[str, Mapping[str, Any]]
+    labels: Union[str, Mapping[str, Any]]
+    reasoning: Optional[str] = None
+    title: Optional[str] = None
+    topic: Optional[str] = None
+    _inputs_needs_transform: bool = field(init=False)
+    _labels_needs_transform: bool = field(init=False)
+
+    def __post_init__(self):
+        self._inputs_needs_transform = isinstance(self.inputs, dict)
+        self._labels_needs_transform = isinstance(self.labels, dict)
+
+    def transform(
+        self,
+        input_transformer: Optional[callable] = None,
+        output_transformer: Optional[callable] = None,
+    ) -> "Example":
+        """Returns a new Example instance with transformed inputs/outputs as strings.
+
+        Args:
+            input_transformer:
+                Function to transform input dict to string.
+            output_transformer:
+                Function to transform output dict to string.
+
+        Returns:
+            New Example instance with string values
+        """
+        new_input = self.inputs
+        new_output = self.labels
+
+        if self._inputs_needs_transform and input_transformer:
+            new_input = input_transformer(self.inputs)
+
+        if self._labels_needs_transform and output_transformer:
+            new_output = output_transformer(self.labels)
+
+        return Example(
+            inputs=new_input,
+            labels=new_output,
+            title=self.title,
+            reasoning=self.reasoning,
+            topic=self.topic,
+        )
+
+    def to_xml(self, example_id: int) -> str:
+        """Convert the example to XML format."""
+        attributes = [f"id={example_id}"]
+
+        if self.title:
+            attributes.append(f'title="{self.title}"')
+        if self.topic:
+            attributes.append(f'topic="{self.topic}"')
+
+        attr_str = " ".join(attributes)
+
+        result = [f"<example {attr_str}>"]
+        result.append(apply_xml_tags("input", str(self.inputs)))
+        if self.reasoning:
+            result.append(apply_xml_tags("reasoning", self.reasoning))
+        result.append(apply_xml_tags("output", str(self.labels)))
+        result.append("</example>")
+
+        return "\n".join(result)
+
+
+class ExampleCollection:
+    def __init__(
+        self, examples: Optional[List[Union[Example, Mapping[str, Any]]]] = None
+    ):
+        self.examples = []
+        if examples:
+            for example in examples:
+                if isinstance(example, Example):
+                    self.examples.append(example)
+                else:
+                    self.add(**example)
+
+    def _add(self, example: Example):
+        self.examples.append(example)
+
+    def add(
+        self,
+        inputs: Union[str, Mapping[str, Any]],
+        labels: Union[str, Mapping[str, Any]],
+        title: Optional[str] = None,
+        reasoning: Optional[str] = None,
+        topic: Optional[str] = None,
+    ):
+        """Add an example to the collection."""
+        example = Example(
+            inputs=inputs,
+            labels=labels,
+            title=title,
+            reasoning=reasoning,
+            topic=topic,
+        )
+        self._add(example)
+
+    def get_examples(self) -> List[Example]:
+        """Get all examples in the collection."""
+        return self.examples
+
+    def filter_by_topic(self, topic: str) -> List[Example]:
+        """Filter examples by topic."""
+        return [ex for ex in self.examples if ex.topic == topic]
+
+    def get_examples_needing_transform(self) -> List[Example]:
+        """Returns examples that need transformation."""
+        return [
+            ex
+            for ex in self.examples
+            if ex._inputs_needs_transform or ex._labels_needs_transform
+        ]
+
+    def transform(
+        self,
+        input_transformer: Optional[callable] = None,
+        output_transformer: Optional[callable] = None,
+    ) -> "ExampleCollection":
+        """Transforms all examples to strings."""
+        transformed = [
+            ex.transform(input_transformer, output_transformer) for ex in self.examples
+        ]
+        return ExampleCollection(transformed)
+
+    def get_formatted(
+        self,
+        input_transformer: Optional[callable] = None,
+        output_transformer: Optional[callable] = None,
+    ) -> str:
+        """Converts the entire collection to XML format.
+
+        Returns:
+            Formatted examples using xml.
+        """
+        if not self.examples:
+            return None
+
+        collection = self.transform(input_transformer, output_transformer)
+
+        xml_parts = []
+        for i, example in enumerate(collection.get_examples(), start=1):
+            xml_parts.append(example.to_xml(i))
+
+        return "\n\n".join(xml_parts)