symbolicai 0.21.0__py3-none-any.whl → 1.1.0__py3-none-any.whl

symai/components.py

@@ -1,23 +1,23 @@
 import copy
 import inspect
 import json
-import os
 import re
 import sys
-from abc import abstractmethod
 from collections import defaultdict
+from collections.abc import Callable, Iterator
 from pathlib import Path
 from random import sample
 from string import ascii_lowercase, ascii_uppercase
 from threading import Lock
-from .context import CURRENT_ENGINE_VAR
-from typing import Callable, Dict, Iterator, List, Optional, Type, Union
+from typing import TYPE_CHECKING, Union
+
+if TYPE_CHECKING:
+    from typing import Any
 
 import numpy as np
-from attr import dataclass
+from beartype import beartype
 from box import Box
 from loguru import logger
-from pydantic import BaseModel, ValidationError
 from pyvis.network import Network
 from tqdm import tqdm
 
@@ -25,46 +25,64 @@ from . import core, core_ext
 from .backend.base import Engine
 from .backend.settings import HOME_PATH
 from .constraints import DictFormatConstraint
+from .context import CURRENT_ENGINE_VAR
 from .formatter import ParagraphFormatter
-from .post_processors import (CodeExtractPostProcessor,
-                              JsonTruncateMarkdownPostProcessor,
-                              JsonTruncatePostProcessor, PostProcessor,
-                              StripPostProcessor)
+from .post_processors import (
+    CodeExtractPostProcessor,
+    JsonTruncateMarkdownPostProcessor,
+    JsonTruncatePostProcessor,
+    PostProcessor,
+    StripPostProcessor,
+)
 from .pre_processors import JsonPreProcessor, PreProcessor
 from .processor import ProcessorPipeline
 from .prompts import JsonPromptTemplate, Prompt
 from .symbol import Expression, Metadata, Symbol
-from .utils import CustomUserWarning
+from .utils import UserMessage
+
+if TYPE_CHECKING:
+    from .backend.engines.index.engine_vectordb import VectorDBResult
+
+_DEFAULT_PARAGRAPH_FORMATTER = ParagraphFormatter()
 
 
 class GraphViz(Expression):
-    def __init__(self,
-                 notebook = True,
-                 cdn_resources = "remote",
-                 bgcolor = "#222222",
-                 font_color = "white",
-                 height = "750px",
-                 width = "100%",
-                 select_menu = True,
-                 filter_menu = True,
-                 **kwargs):
+    def __init__(
+        self,
+        notebook=True,
+        cdn_resources="remote",
+        bgcolor="#222222",
+        font_color="white",
+        height="750px",
+        width="100%",
+        select_menu=True,
+        filter_menu=True,
+        **kwargs,
+    ):
         super().__init__(**kwargs)
-        self.net  = Network(notebook=notebook,
-                            cdn_resources=cdn_resources,
-                            bgcolor=bgcolor,
-                            font_color=font_color,
-                            height=height,
-                            width=width,
-                            select_menu=select_menu,
-                            filter_menu=filter_menu)
-
-    def forward(self, sym: Symbol, file_path: str, **kwargs):
+        self.net = Network(
+            notebook=notebook,
+            cdn_resources=cdn_resources,
+            bgcolor=bgcolor,
+            font_color=font_color,
+            height=height,
+            width=width,
+            select_menu=select_menu,
+            filter_menu=filter_menu,
+        )
+
+    def forward(self, sym: Symbol, file_path: str, **_kwargs):
         nodes = [str(n) if n.value else n.__repr__(simplified=True) for n in sym.nodes]
-        edges = [(str(e[0]) if e[0].value else e[0].__repr__(simplified=True),
-                  str(e[1]) if e[1].value else e[1].__repr__(simplified=True)) for e in sym.edges]
+        edges = [
+            (
+                str(e[0]) if e[0].value else e[0].__repr__(simplified=True),
+                str(e[1]) if e[1].value else e[1].__repr__(simplified=True),
+            )
+            for e in sym.edges
+        ]
         self.net.add_nodes(nodes)
         self.net.add_edges(edges)
-        file_path = file_path if file_path.endswith('.html') else file_path + '.html'
+        file_path = file_path if file_path.endswith(".html") else file_path + ".html"
         return self.net.show(file_path)
 
 
@@ -73,21 +91,21 @@ class TrackerTraceable(Expression):
 
 
 class Any(Expression):
-    def __init__(self, *expr: List[Expression], **kwargs):
+    def __init__(self, *expr: list[Expression], **kwargs):
         super().__init__(**kwargs)
-        self.expr: List[Expression] = expr
+        self.expr: list[Expression] = expr
 
     def forward(self, *args, **kwargs) -> Symbol:
-        return self.sym_return_type(any([e() for e in self.expr(*args, **kwargs)]))
+        return self.sym_return_type(any(e() for e in self.expr(*args, **kwargs)))
 
 
 class All(Expression):
-    def __init__(self, *expr: List[Expression], **kwargs):
+    def __init__(self, *expr: list[Expression], **kwargs):
         super().__init__(**kwargs)
-        self.expr: List[Expression] = expr
+        self.expr: list[Expression] = expr
 
     def forward(self, *args, **kwargs) -> Symbol:
-        return self.sym_return_type(all([e() for e in self.expr(*args, **kwargs)]))
+        return self.sym_return_type(all(e() for e in self.expr(*args, **kwargs)))
 
 
 class Try(Expression):
@@ -104,12 +122,14 @@ class Try(Expression):
 class Lambda(Expression):
     def __init__(self, callable: Callable, **kwargs):
         super().__init__(**kwargs)
+
         def _callable(*args, **kwargs):
             kw = {
-                'args': args,
-                'kwargs': kwargs,
+                "args": args,
+                "kwargs": kwargs,
             }
             return callable(kw)
+
         self.callable: Callable = _callable
 
     def forward(self, *args, **kwargs) -> Symbol:
@@ -117,14 +137,14 @@ class Lambda(Expression):
 
 
 class Choice(Expression):
-    def __init__(self, cases: List[str], default: Optional[str] = None, **kwargs):
+    def __init__(self, cases: list[str], default: str | None = None, **kwargs):
         super().__init__(**kwargs)
-        self.cases: List[str] = cases
-        self.default: Optional[str] = default
+        self.cases: list[str] = cases
+        self.default: str | None = default
 
     def forward(self, sym: Symbol, *args, **kwargs) -> Symbol:
         sym = self._to_symbol(sym)
-        return sym.choice(cases=self.cases, default=self.default, *args, **kwargs)
+        return sym.choice(*args, cases=self.cases, default=self.default, **kwargs)
 
 
 class Output(Expression):
@@ -135,15 +155,15 @@ class Output(Expression):
         self.verbose: bool = verbose
 
     def forward(self, *args, **kwargs) -> Expression:
-        kwargs['verbose'] = self.verbose
-        kwargs['handler'] = self.handler
-        return self.output(expr=self.expr, *args, **kwargs)
+        kwargs["verbose"] = self.verbose
+        kwargs["handler"] = self.handler
+        return self.output(*args, expr=self.expr, **kwargs)
 
 
 class Sequence(TrackerTraceable):
-    def __init__(self, *expressions: List[Expression], **kwargs):
+    def __init__(self, *expressions: list[Expression], **kwargs):
         super().__init__(**kwargs)
-        self.expressions: List[Expression] = expressions
+        self.expressions: list[Expression] = expressions
 
     def forward(self, *args, **kwargs) -> Symbol:
         sym = self.expressions[0](*args, **kwargs)
@@ -159,34 +179,36 @@ class Sequence(TrackerTraceable):
 
 
 class Parallel(Expression):
-    def __init__(self, *expr: List[Expression | Callable], sequential: bool = False, **kwargs):
+    def __init__(self, *expr: list[Expression | Callable], sequential: bool = False, **kwargs):
         super().__init__(**kwargs)
-        self.sequential: bool       = sequential
-        self.expr: List[Expression] = expr
-        self.results: List[Symbol]  = []
+        self.sequential: bool = sequential
+        self.expr: list[Expression] = expr
+        self.results: list[Symbol] = []
 
     def forward(self, *args, **kwargs) -> Symbol:
         # run in sequence
         if self.sequential:
             return [e(*args, **kwargs) for e in self.expr]
+
         # run in parallel
         @core_ext.parallel(self.expr)
         def _func(e, *args, **kwargs):
             return e(*args, **kwargs)
+
         self.results = _func(*args, **kwargs)
         # final result of the parallel execution
         return self._to_symbol(self.results)
 
 
-#@TODO: BinPacker(format="...") -> ensure that data packages form a "bin" that's consistent (e.g. never break a sentence in the middle)
+# @TODO: BinPacker(format="...") -> ensure that data packages form a "bin" that's consistent (e.g. never break a sentence in the middle)
 class Stream(Expression):
-    def __init__(self, expr: Optional[Expression] = None, retrieval: Optional[str] = None, **kwargs):
+    def __init__(self, expr: Expression | None = None, retrieval: str | None = None, **kwargs):
         super().__init__(**kwargs)
-        self.char_token_ratio:    float = 0.6
-        self.expr: Optional[Expression] = expr
-        self.retrieval:   Optional[str] = retrieval
-        self._trace:               bool = False
-        self._previous_frame            = None
+        self.char_token_ratio: float = 0.6
+        self.expr: Expression | None = expr
+        self.retrieval: str | None = retrieval
+        self._trace: bool = False
+        self._previous_frame = None
 
     def forward(self, sym: Symbol, **kwargs) -> Iterator:
         sym = self._to_symbol(sym)
@@ -194,30 +216,31 @@ class Stream(Expression):
         if self._trace:
             local_vars = self._previous_frame.f_locals
             vals = []
-            for key, var in local_vars.items():
+            for _key, var in local_vars.items():
                 if isinstance(var, TrackerTraceable):
                     vals.append(var)
 
             if len(vals) == 1:
                 self.expr = vals[0]
             else:
-                raise ValueError(f"This component does either not inherit from TrackerTraceable or has an invalid number of component declarations: {len(vals)}! Only one component that inherits from TrackerTraceable is allowed in the with stream clause.")
-
-        res = sym.stream(expr=self.expr,
-                         char_token_ratio=self.char_token_ratio,
-                         **kwargs)
-
+                UserMessage(
+                    "This component does either not inherit from TrackerTraceable or has an invalid number of component "
+                    f"declarations: {len(vals)}! Only one component that inherits from TrackerTraceable is allowed in the "
+                    "with stream clause.",
+                    raise_with=ValueError,
+                )
+
+        res = sym.stream(expr=self.expr, char_token_ratio=self.char_token_ratio, **kwargs)
         if self.retrieval is not None:
             res = list(res)
-            if self.retrieval == 'all':
+            if self.retrieval == "all":
                 return res
-            if self.retrieval == 'longest':
+            if self.retrieval == "longest":
                 res = sorted(res, key=lambda x: len(x), reverse=True)
                 return res[0]
-            if self.retrieval == 'contains':
-                res = [r for r in res if self.expr in r]
-                return res
-            raise ValueError(f"Invalid retrieval method: {self.retrieval}")
+            if self.retrieval == "contains":
+                return [r for r in res if self.expr in r]
+            UserMessage(f"Invalid retrieval method: {self.retrieval}", raise_with=ValueError)
 
         return res
 
@@ -231,10 +254,12 @@ class Stream(Expression):
 
 
 class Trace(Expression):
-    def __init__(self, expr: Optional[Expression] = None, engines=['all'], **kwargs):
+    def __init__(self, expr: Expression | None = None, engines=None, **kwargs):
+        if engines is None:
+            engines = ["all"]
         super().__init__(**kwargs)
         self.expr: Expression = expr
-        self.engines: List[str] = engines
+        self.engines: list[str] = engines
 
     def forward(self, *args, **kwargs) -> Expression:
         Expression.command(verbose=True, engines=self.engines)
@@ -252,23 +277,26 @@ class Trace(Expression):
         Expression.command(verbose=False, engines=self.engines)
         if self.expr is not None:
             return self.expr.__exit__(type, value, traceback)
+        return None
 
 
 class Analyze(Expression):
-    def __init__(self, exception: Exception, query: Optional[str] = None, **kwargs):
+    def __init__(self, exception: Exception, query: str | None = None, **kwargs):
         super().__init__(**kwargs)
         self.exception: Expression = exception
-        self.query: Optional[str] = query
+        self.query: str | None = query
 
     def forward(self, sym: Symbol, *args, **kwargs) -> Symbol:
-        return sym.analyze(exception=self.exception, query=self.query, *args, **kwargs)
+        return sym.analyze(*args, exception=self.exception, query=self.query, **kwargs)
 
 
 class Log(Expression):
-    def __init__(self, expr: Optional[Expression] = None, engines=['all'], **kwargs):
+    def __init__(self, expr: Expression | None = None, engines=None, **kwargs):
+        if engines is None:
+            engines = ["all"]
         super().__init__(**kwargs)
         self.expr: Expression = expr
-        self.engines: List[str] = engines
+        self.engines: list[str] = engines
 
     def forward(self, *args, **kwargs) -> Expression:
         Expression.command(logging=True, engines=self.engines)
@@ -286,10 +314,16 @@ class Log(Expression):
         Expression.command(logging=False, engines=self.engines)
         if self.expr is not None:
             return self.expr.__exit__(type, value, traceback)
+        return None
 
 
 class Template(Expression):
-    def __init__(self, template: str = "<html><body>{{placeholder}}</body></html>", placeholder: str = '{{placeholder}}', **kwargs):
+    def __init__(
+        self,
+        template: str = "<html><body>{{placeholder}}</body></html>",
+        placeholder: str = "{{placeholder}}",
+        **kwargs,
+    ):
         super().__init__(**kwargs)
         self.placeholder = placeholder
         self.template_ = template
@@ -319,22 +353,26 @@ class RuntimeExpression(Expression):
         code = self._to_symbol(code)
         # declare the runtime expression from the code
         expr = self.runner(code)
+
         def _func(sym):
             # execute nested expression
-            return expr['locals']['_output_'](sym)
+            return expr["locals"]["_output_"](sym)
+
         return _func
 
 
 class Metric(Expression):
     def __init__(self, normalize: bool = False, eps: float = 1e-8, **kwargs):
         super().__init__(**kwargs)
-        self.normalize  = normalize
-        self.eps        = eps
+        self.normalize = normalize
+        self.eps = eps
 
-    def forward(self, sym: Symbol, **kwargs) -> Symbol:
+    def forward(self, sym: Symbol, **_kwargs) -> Symbol:
         sym = self._to_symbol(sym)
-        assert sym.value_type == np.ndarray or sym.value_type == list, 'Metric can only be applied to numpy arrays or lists.'
-        if sym.value_type == list:
+        assert sym.value_type is np.ndarray or sym.value_type is list, (
+            "Metric can only be applied to numpy arrays or lists."
+        )
+        if sym.value_type is list:
             sym._value = np.array(sym.value)
         # compute normalization between 0 and 1
         if self.normalize:
@@ -343,17 +381,19 @@ class Metric(Expression):
             elif len(sym.value.shape) == 2:
                 pass
             else:
-                raise ValueError(f'Invalid shape: {sym.value.shape}')
+                UserMessage(f"Invalid shape: {sym.value.shape}", raise_with=ValueError)
             # normalize between 0 and 1 and sum to 1
             sym._value = np.exp(sym.value) / (np.exp(sym.value).sum() + self.eps)
         return sym
 
 
 class Style(Expression):
-    def __init__(self, description: str, libraries: List[str] = [], **kwargs):
+    def __init__(self, description: str, libraries: list[str] | None = None, **kwargs):
+        if libraries is None:
+            libraries = []
         super().__init__(**kwargs)
         self.description: str = description
-        self.libraries: List[str] = libraries
+        self.libraries: list[str] = libraries
 
     def forward(self, sym: Symbol, **kwargs) -> Symbol:
         sym = self._to_symbol(sym)
@@ -365,7 +405,7 @@ class Query(TrackerTraceable):
         super().__init__(**kwargs)
         self.prompt: str = prompt
 
-    def forward(self, sym: Symbol, context: Symbol = None, *args, **kwargs) -> Symbol:
+    def forward(self, sym: Symbol, context: Symbol = None, *_args, **kwargs) -> Symbol:
         sym = self._to_symbol(sym)
         return sym.query(prompt=self.prompt, context=context, **kwargs)
 
@@ -397,16 +437,16 @@ _output_ = _func()
 
     def forward(self, sym: Symbol, enclosure: bool = False, **kwargs) -> Symbol:
         if enclosure or self.enclosure:
-            lines = str(sym).split('\n')
-            lines = ['    ' + line for line in lines]
-            sym = '\n'.join(lines)
-            sym = self.template.replace('{sym}', str(sym))
+            lines = str(sym).split("\n")
+            lines = ["    " + line for line in lines]
+            sym = "\n".join(lines)
+            sym = self.template.replace("{sym}", str(sym))
         sym = self._to_symbol(sym)
         return sym.execute(**kwargs)
 
 
 class Convert(Expression):
-    def __init__(self, format: str = 'Python', **kwargs):
+    def __init__(self, format: str = "Python", **kwargs):
         super().__init__(**kwargs)
         self.format = format
 
@@ -440,13 +480,13 @@ class Map(Expression):
 
 
 class Translate(Expression):
-    def __init__(self, language: str = 'English', **kwargs):
+    def __init__(self, language: str = "English", **kwargs):
         super().__init__(**kwargs)
         self.language = language
 
     def forward(self, sym: Symbol, **kwargs) -> Symbol:
         sym = self._to_symbol(sym)
-        if sym.isinstanceof(f'{self.language} text'):
+        if sym.isinstanceof(f"{self.language} text"):
             return sym
         return sym.translate(language=self.language, **kwargs)
 
@@ -474,11 +514,11 @@ class ExcludeFilter(Expression):
 class FileWriter(Expression):
     def __init__(self, path: str, **kwargs):
         super().__init__(**kwargs)
-        self.path = path
+        self.path = Path(path)
 
-    def forward(self, sym: Symbol, **kwargs) -> Symbol:
+    def forward(self, sym: Symbol, **_kwargs) -> Symbol:
         sym = self._to_symbol(sym)
-        with open(self.path, 'w') as f:
+        with self.path.open("w") as f:
             f.write(str(sym))
 
 
@@ -486,20 +526,18 @@ class FileReader(Expression):
     @staticmethod
     def exists(path: str) -> bool:
         # remove slicing if any
-        _tmp     = path
-        _splits  = _tmp.split('[')
-        if '[' in _tmp:
+        _tmp = path
+        _splits = _tmp.split("[")
+        if "[" in _tmp:
             _tmp = _splits[0]
-        assert len(_splits) == 1 or len(_splits) == 2, 'Invalid file link format.'
-        _tmp     = Path(_tmp)
+        assert len(_splits) == 1 or len(_splits) == 2, "Invalid file link format."
+        _tmp = Path(_tmp)
         # check if file exists and is a file
-        if os.path.exists(_tmp) and os.path.isfile(_tmp):
-            return True
-        return False
+        return _tmp.is_file()
 
     @staticmethod
-    def get_files(folder_path: str, max_depth: int = 1) -> List[str]:
-        accepted_formats = ['.pdf', '.md', '.txt']
+    def get_files(folder_path: str, max_depth: int = 1) -> list[str]:
+        accepted_formats = [".pdf", ".md", ".txt"]
 
         folder = Path(folder_path)
         files = []
@@ -512,10 +550,35 @@ class FileReader(Expression):
         return files
 
     @staticmethod
-    def extract_files(cmds: str) -> Optional[List[str]]:
-        # Use the updated regular expression to match quoted and non-quoted paths
-        pattern = r'''(?:"((?:\\.|[^"\\])*)"|'((?:\\.|[^'\\])*)'|`((?:\\.|[^`\\])*)`|((?:\\ |[^ ])+))'''
-        # Use the regular expression to split and handle quoted and non-quoted paths
+    def extract_files(cmds: str) -> list[str] | None:
+        """
+        Extract file paths from a command string, handling various quoting styles.
+
+        This method is used by the Qdrant RAG implementation when processing document paths.
+        It uses regex to parse file paths that may be quoted in different ways.
+
+        Regex patterns used:
+        1. Main pattern: Matches file paths in four formats:
+           - Double-quoted: "path/to/file" (handles escaped characters)
+           - Single-quoted: 'path/to/file' (handles escaped characters)
+           - Backtick-quoted: `path/to/file` (handles escaped characters)
+           - Non-quoted: path/to/file (handles escaped spaces)
+
+        2. Escape removal pattern: r"\\(.)" -> r"\1"
+           - Removes backslash escape sequences from quoted paths
+           - Example: "path\\/to\\/file" -> "path/to/file"
+           - Used for double quotes, single quotes, and backticks
+        """
+        # Regex pattern to match file paths in various quoting styles
+        # Pattern breakdown:
+        # - (?:"((?:\\.|[^"\\])*)") : Matches double-quoted paths, capturing content while handling escapes
+        # - '((?:\\.|[^'\\])*)' : Matches single-quoted paths, capturing content while handling escapes
+        # - `((?:\\.|[^`\\])*)` : Matches backtick-quoted paths, capturing content while handling escapes
+        # - ((?:\\ |[^ ])+) : Matches non-quoted paths, allowing escaped spaces
+        pattern = (
+            r"""(?:"((?:\\.|[^"\\])*)"|'((?:\\.|[^'\\])*)'|`((?:\\.|[^`\\])*)`|((?:\\ |[^ ])+))"""
+        )
+        # Use regex to find all file path matches in the command string
         matches = re.findall(pattern, cmds)
         # Process the matches to handle quoted paths and normal paths
         files = []
@@ -523,23 +586,27 @@ class FileReader(Expression):
             # Each match will have 4 groups due to the pattern; only one will be non-empty
             quoted_double, quoted_single, quoted_backtick, non_quoted = match
             if quoted_double:
-                # Remove backslashes used for escaping inside double quotes
-                path = re.sub(r'\\(.)', r'\1', quoted_double)
+                # Regex substitution: Remove backslashes used for escaping inside double quotes
+                # Pattern r"\\(.)" matches a backslash followed by any character and replaces with just the character
+                # Example: "path\\/to\\/file" -> "path/to/file"
+                path = re.sub(r"\\(.)", r"\1", quoted_double)
                 file = FileReader.expand_user_path(path)
                 files.append(file)
             elif quoted_single:
-                # Remove backslashes used for escaping inside single quotes
-                path = re.sub(r'\\(.)', r'\1', quoted_single)
+                # Regex substitution: Remove backslashes used for escaping inside single quotes
+                # Same pattern as above, applied to single-quoted paths
+                path = re.sub(r"\\(.)", r"\1", quoted_single)
                 file = FileReader.expand_user_path(path)
                 files.append(file)
             elif quoted_backtick:
-                # Remove backslashes used for escaping inside backticks
-                path = re.sub(r'\\(.)', r'\1', quoted_backtick)
+                # Regex substitution: Remove backslashes used for escaping inside backticks
+                # Same pattern as above, applied to backtick-quoted paths
+                path = re.sub(r"\\(.)", r"\1", quoted_backtick)
                 file = FileReader.expand_user_path(path)
                 files.append(file)
             elif non_quoted:
-                # Replace escaped spaces with actual spaces
-                path = non_quoted.replace('\\ ', ' ')
+                # Replace escaped spaces with actual spaces (no regex needed here, simple string replace)
+                path = non_quoted.replace("\\ ", " ")
                 file = FileReader.expand_user_path(path)
                 files.append(file)
         # Filter out any files that do not exist
@@ -551,31 +618,34 @@ class FileReader(Expression):
         return Path(path).expanduser().resolve().as_posix()
 
     @staticmethod
-    def integrity_check(files: List[str]) -> List[str]:
+    def integrity_check(files: list[str]) -> list[str]:
         not_skipped = []
         for file in tqdm(files):
             if FileReader.exists(file):
                 not_skipped.append(file)
             else:
-                CustomUserWarning(f'Skipping file: {file}')
+                UserMessage(f"Skipping file: {file}")
         return not_skipped
 
-    def forward(self, files: Union[str, List[str]], **kwargs) -> Expression:
+    def forward(self, files: str | list[str], **kwargs) -> Expression:
         if isinstance(files, str):
             # Convert to list for uniform processing; more easily downstream
             files = [files]
-        if kwargs.get('run_integrity_check'):
+        if kwargs.get("run_integrity_check"):
             files = self.integrity_check(files)
         return self.sym_return_type([self.open(f, **kwargs).value for f in files])
 
+
 class FileQuery(Expression):
     def __init__(self, path: str, filter: str, **kwargs):
         super().__init__(**kwargs)
         self.path = path
         file_open = FileReader()
-        self.query_stream = Stream(Sequence(
-            IncludeFilter(filter),
-        ))
+        self.query_stream = Stream(
+            Sequence(
+                IncludeFilter(filter),
+            )
+        )
         self.file = file_open(path)
 
     def forward(self, sym: Symbol, **kwargs) -> Symbol:
@@ -585,40 +655,45 @@ class FileQuery(Expression):
 
 
 class Function(TrackerTraceable):
-    def __init__(self, prompt: str       = '',
-                 examples: Optional[str] = [],
-                 pre_processors: Optional[List[PreProcessor]]   = None,
-                 post_processors: Optional[List[PostProcessor]] = None,
-                 default: Optional[object]       = None,
-                 constraints: List[Callable]     = [],
-                 return_type: Optional[Type]     = str,
-                 sym_return_type: Optional[Type] = Symbol,
-                 origin_type: Optional[Type]     = Expression,
-                 *args, **kwargs):
+    def __init__(
+        self,
+        prompt: str = "",
+        examples: str | None = [],
+        pre_processors: list[PreProcessor] | None = None,
+        post_processors: list[PostProcessor] | None = None,
+        default: object | None = None,
+        constraints: list[Callable] | None = None,
+        return_type: type | None = str,
+        sym_return_type: type | None = Symbol,
+        origin_type: type | None = Expression,
+        *args,
+        **kwargs,
+    ):
+        if constraints is None:
+            constraints = []
         super().__init__(**kwargs)
-        chars       = ascii_lowercase + ascii_uppercase
-        self.name   = 'func_' + ''.join(sample(chars, 15))
-        self.args   = args
+        chars = ascii_lowercase + ascii_uppercase
+        self.name = "func_" + "".join(sample(chars, 15))
+        self.args = args
         self.kwargs = kwargs
-        self._promptTemplate     = prompt
-        self._promptFormatArgs   = []
+        self._promptTemplate = prompt
+        self._promptFormatArgs = []
         self._promptFormatKwargs = {}
-        self.examples        = Prompt(examples)
-        self.pre_processors  = pre_processors
+        self.examples = Prompt(examples)
+        self.pre_processors = pre_processors
         self.post_processors = post_processors
-        self.constraints     = constraints
-        self.default         = default
-        self.return_type     = return_type
+        self.constraints = constraints
+        self.default = default
+        self.return_type = return_type
         self.sym_return_type = sym_return_type
-        self.origin_type     = origin_type
+        self.origin_type = origin_type
 
     @property
     def prompt(self):
         # return a copy of the prompt template
         if len(self._promptFormatArgs) == 0 and len(self._promptFormatKwargs) == 0:
             return self._promptTemplate
-        return f"{self._promptTemplate}".format(*self._promptFormatArgs,
-                                                **self._promptFormatKwargs)
+        return f"{self._promptTemplate}".format(*self._promptFormatArgs, **self._promptFormatKwargs)
 
     def format(self, *args, **kwargs):
         self._promptFormatArgs = args
@@ -626,27 +701,36 @@ class Function(TrackerTraceable):
 
     def forward(self, *args, **kwargs) -> Expression:
         # special case for few shot function prompt definition override
-        if 'fn' in kwargs:
-            self.prompt = kwargs['fn']
-            del kwargs['fn']
-        @core.few_shot(prompt=self.prompt,
-                  examples=self.examples,
-                  pre_processors=self.pre_processors,
-                  post_processors=self.post_processors,
-                  constraints=self.constraints,
-                  default=self.default,
-                  *self.args, **self.kwargs)
+        if "fn" in kwargs:
+            self.prompt = kwargs["fn"]
+            del kwargs["fn"]
+
+        @core.few_shot(
+            *self.args,
+            prompt=self.prompt,
+            examples=self.examples,
+            pre_processors=self.pre_processors,
+            post_processors=self.post_processors,
+            constraints=self.constraints,
+            default=self.default,
+            **self.kwargs,
+        )
         def _func(_, *args, **kwargs) -> self.return_type:
             pass
-        _type = type(self.name, (self.origin_type, ), {
-            # constructor
-            "forward": _func,
-            "sym_return_type": self.sym_return_type,
-            "static_context": self.static_context,
-            "dynamic_context": self.dynamic_context,
-            "__class__": self.__class__,
-            "__module__": self.__module__,
-        })
+
+        _type = type(
+            self.name,
+            (self.origin_type,),
+            {
+                # constructor
+                "forward": _func,
+                "sym_return_type": self.sym_return_type,
+                "static_context": self.static_context,
+                "dynamic_context": self.dynamic_context,
+                "__class__": self.__class__,
+                "__module__": self.__module__,
+            },
+        )
         obj = _type()
 
         return self._to_symbol(obj(*args, **kwargs))
@@ -657,19 +741,19 @@ class PrepareData(Function):
         def __call__(self, argument):
             assert argument.prop.context is not None
             instruct = argument.prop.prompt
-            context  = argument.prop.context
-            return """{
-    'context': '%s',
-    'instruction': '%s',
+            context = argument.prop.context
+            return f"""{{
+    'context': '{context}',
+    'instruction': '{instruct}',
     'result': 'TODO: Replace this with the expected result.'
-}""" % (context, instruct)
+}}"""
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self.pre_processors  = [self.PrepareDataPreProcessor()]
-        self.constraints     = [DictFormatConstraint({ 'result': '<the data>' })]
+        self.pre_processors = [self.PrepareDataPreProcessor()]
+        self.constraints = [DictFormatConstraint({"result": "<the data>"})]
         self.post_processors = [JsonTruncateMarkdownPostProcessor()]
-        self.return_type     = dict # constraint to cast the result to a dict
+        self.return_type = dict  # constraint to cast the result to a dict
 
     @property
     def static_context(self):
@@ -704,10 +788,10 @@ Your goal is to prepare the data for the next task instruction. The data should
 
 class ExpressionBuilder(Function):
     def __init__(self, **kwargs):
-        super().__init__('Generate the code following the instructions:', **kwargs)
+        super().__init__("Generate the code following the instructions:", **kwargs)
         self.processors = ProcessorPipeline([StripPostProcessor(), CodeExtractPostProcessor()])
 
-    def forward(self, instruct, *args, **kwargs):
+    def forward(self, instruct, *_args, **_kwargs):
         result = super().forward(instruct)
         return self.processors(str(result), None)
 
@@ -755,10 +839,12 @@ Always produce the entire code to be executed in the same Python process. All ta
 class JsonParser(Expression):
     def __init__(self, query: str, json_: dict, **kwargs):
         super().__init__(**kwargs)
-        func = Function(prompt=JsonPromptTemplate(query, json_),
-                        constraints=[DictFormatConstraint(json_)],
-                        pre_processors=[JsonPreProcessor()],
-                        post_processors=[JsonTruncatePostProcessor()])
+        func = Function(
+            prompt=JsonPromptTemplate(query, json_),
+            constraints=[DictFormatConstraint(json_)],
+            pre_processors=[JsonPreProcessor()],
+            post_processors=[JsonTruncatePostProcessor()],
+        )
         self.fn = Try(func, retries=1)
 
     def forward(self, sym: Symbol, **kwargs) -> Symbol:
@@ -768,21 +854,27 @@ class JsonParser(Expression):
 
 
 class SimilarityClassification(Expression):
-    def __init__(self, classes: List[str], metric: str = 'cosine', in_memory: bool = False, **kwargs):
+    def __init__(
+        self, classes: list[str], metric: str = "cosine", in_memory: bool = False, **kwargs
+    ):
         super().__init__(**kwargs)
-        self.classes   = classes
-        self.metric    = metric
+        self.classes = classes
+        self.metric = metric
         self.in_memory = in_memory
 
         if self.in_memory:
-            CustomUserWarning(f'Caching mode is enabled! It is your responsability to empty the .cache folder if you did changes to the classes. The cache is located at {HOME_PATH}/cache')
+            UserMessage(
+                f"Caching mode is enabled! It is your responsability to empty the .cache folder if you did changes to the classes. The cache is located at {HOME_PATH}/cache"
+            )
 
     def forward(self, x: Symbol) -> Symbol:
-        x            = self._to_symbol(x)
-        usr_embed    = x.embed()
-        embeddings   = self._dynamic_cache()
+        x = self._to_symbol(x)
+        usr_embed = x.embed()
+        embeddings = self._dynamic_cache()
         similarities = [usr_embed.similarity(emb, metric=self.metric) for emb in embeddings]
-        similarities = sorted(zip(self.classes, similarities), key=lambda x: x[1], reverse=True)
+        similarities = sorted(
+            zip(self.classes, similarities, strict=False), key=lambda x: x[1], reverse=True
+        )
 
         return Symbol(similarities[0][0])
 
@@ -790,9 +882,7 @@ class SimilarityClassification(Expression):
         @core_ext.cache(in_memory=self.in_memory)
         def embed_classes(self):
             opts = map(Symbol, self.classes)
-            embeddings = [opt.embed() for opt in opts]
-
-            return embeddings
+            return [opt.embed() for opt in opts]
 
         return embed_classes(self)
 
@@ -803,11 +893,7 @@ class InContextClassification(Expression):
         self.blueprint = blueprint
 
     def forward(self, x: Symbol, **kwargs) -> Symbol:
-        @core.few_shot(
-            prompt=x,
-            examples=self.blueprint,
-            **kwargs
-        )
+        @core.few_shot(prompt=x, examples=self.blueprint, **kwargs)
         def _func(_):
             pass
 
@@ -815,43 +901,38 @@ class InContextClassification(Expression):
 
 
 class Indexer(Expression):
-    DEFAULT = 'dataindex'
+    DEFAULT = "dataindex"
 
     @staticmethod
     def replace_special_chars(index: str):
         # replace special characters that are not for path
-        index = str(index)
-        index = index.replace('-', '')
-        index = index.replace('_', '')
-        index = index.replace(' ', '')
-        index = index.lower()
-        return index
+        return str(index).replace("-", "").replace("_", "").replace(" ", "").lower()
 
     def __init__(
-            self,
-            index_name: str = DEFAULT,
-            top_k: int = 8,
-            batch_size: int = 20,
-            formatter: Callable = ParagraphFormatter(),
-            auto_add=False,
-            raw_result: bool = False,
-            new_dim: int = 1536,
-            **kwargs
-        ):
+        self,
+        index_name: str = DEFAULT,
+        top_k: int = 8,
+        batch_size: int = 20,
+        formatter: Callable = _DEFAULT_PARAGRAPH_FORMATTER,
+        auto_add=False,
+        raw_result: bool = False,
+        new_dim: int = 1536,
+        **kwargs,
+    ):
         super().__init__(**kwargs)
         index_name = Indexer.replace_special_chars(index_name)
         self.index_name = index_name
-        self.elements   = []
+        self.elements = []
         self.batch_size = batch_size
-        self.top_k      = top_k
-        self.retrieval  = None
-        self.formatter  = formatter
+        self.top_k = top_k
+        self.retrieval = None
+        self.formatter = formatter
         self.raw_result = raw_result
-        self.new_dim    = new_dim
+        self.new_dim = new_dim
         self.sym_return_type = Expression
 
         # append index name to indices.txt in home directory .symai folder (default)
-        self.path = HOME_PATH / 'indices.txt'
+        self.path = HOME_PATH / "indices.txt"
         if not self.path.exists():
             self.path.parent.mkdir(parents=True, exist_ok=True)
             self.path.touch()
@@ -861,52 +942,63 @@ class Indexer(Expression):
     def register(self):
         # check if index already exists in indices.txt and append if not
         change = False
-        with open(self.path, 'r') as f:
-            indices = f.read().split('\n')
+        with self.path.open() as f:
+            indices = f.read().split("\n")
             # filter out empty strings
             indices = [i for i in indices if i]
-            if self.index_name not in indices:
-                indices.append(self.index_name)
-                change = True
+        if self.index_name not in indices:
+            indices.append(self.index_name)
+            change = True
         if change:
-            with open(self.path, 'w') as f:
-                f.write('\n'.join(indices))
+            with self.path.open("w") as f:
+                f.write("\n".join(indices))
 
     def exists(self) -> bool:
         # check if index exists in home directory .symai folder (default) indices.txt
-        path = HOME_PATH / 'indices.txt'
+        path = HOME_PATH / "indices.txt"
         if not path.exists():
             return False
-        with open(path, 'r') as f:
-            indices = f.read().split('\n')
+        with path.open() as f:
+            indices = f.read().split("\n")
             if self.index_name in indices:
                 return True
+        return False
 
     def forward(
-            self,
-            data: Optional[Symbol] = None,
-            raw_result: bool = False,
-        ) -> Symbol:
+        self,
+        data: Symbol | None = None,
+        _raw_result: bool = False,
+    ) -> Symbol:
         that = self
         if data is not None:
             data = self._to_symbol(data)
             self.elements = self.formatter(data).value
             # run over the elments in batches
             for i in tqdm(range(0, len(self.elements), self.batch_size)):
-                val = Symbol(self.elements[i:i+self.batch_size]).zip(new_dim=self.new_dim)
+                val = Symbol(self.elements[i : i + self.batch_size]).zip(new_dim=self.new_dim)
                 that.add(val, index_name=that.index_name, index_dims=that.new_dim)
             # we save the index
             that.config(None, save=True, index_name=that.index_name, index_dims=that.new_dim)
 
-        def _func(query, *args, **kwargs) -> Union[Symbol, 'VectorDBResult']:
-            raw_result = kwargs.get('raw_result') or that.raw_result
+        def _func(query, *_args, **kwargs) -> Union[Symbol, "VectorDBResult"]:
+            raw_result = kwargs.get("raw_result") or that.raw_result
             query_emb = Symbol(query).embed(new_dim=that.new_dim).value
-            res = that.get(query_emb, index_name=that.index_name, index_top_k=that.top_k, ori_query=query, index_dims=that.new_dim, **kwargs)
+            res = that.get(
+                query_emb,
+                index_name=that.index_name,
+                index_top_k=that.top_k,
+                ori_query=query,
+                index_dims=that.new_dim,
+                **kwargs,
+            )
             that.retrieval = res
             if raw_result:
                 return res
-            rsp = Symbol(res).query(prompt='From the retrieved data, select the most relevant information.', context=query)
-            return rsp
+            return Symbol(res).query(
+                prompt="From the retrieved data, select the most relevant information.",
+                context=query,
+            )
+
         return _func
 
 
@@ -917,8 +1009,8 @@ class PrimitiveDisabler(Expression):
         self._original_primitives = defaultdict(list)
 
     def __enter__(self):
-        # Avoid circular imports; import locally
-        from .symbol import Symbol
+        # Import Symbol lazily so components does not clash with symbol during load.
+        from .symbol import Symbol  # noqa
 
         frame = inspect.currentframe()
         f_locals = frame.f_back.f_locals
@@ -934,7 +1026,7 @@ class PrimitiveDisabler(Expression):
             for func in self._primitives:
                 if hasattr(sym, func):
                     self._original_primitives[sym_name].append((func, getattr(sym, func)))
-                    setattr(sym, func, lambda *args, **kwargs: None)
+                    setattr(sym, func, lambda *_args, **_kwargs: None)
 
     def _enable_primitives(self):
         for sym_name, sym in self._symbols.items():
@@ -945,7 +1037,7 @@ class PrimitiveDisabler(Expression):
         for sym in self._symbols.values():
             for primitive in sym._primitives:
                 for method, _ in inspect.getmembers(primitive, predicate=inspect.isfunction):
-                    if method in self._primitives or method.startswith('_'):
+                    if method in self._primitives or method.startswith("_"):
                         continue
                     self._primitives.add(method)
 
@@ -968,7 +1060,7 @@ class FunctionWithUsage(Function):
 
     def print_verbose(self, msg):
         if self.verbose:
-            print(msg)
+            UserMessage(msg)
 
     def _format_usage(self, prompt_tokens, completion_tokens, total_tokens):
         return Box(
@@ -990,9 +1082,7 @@ class FunctionWithUsage(Function):
         self.total_tokens += usage.total_tokens
 
     def get_usage(self):
-        return self._format_usage(
-            self.prompt_tokens, self.completion_tokens, self.total_tokens
-        )
+        return self._format_usage(self.prompt_tokens, self.completion_tokens, self.total_tokens)
 
     def forward(self, *args, **kwargs):
         if "return_metadata" not in kwargs:
@@ -1003,9 +1093,7 @@ class FunctionWithUsage(Function):
         raw_output = metadata.get("raw_output")
         if hasattr(raw_output, "usage"):
             usage = raw_output.usage
-            prompt_tokens = (
-                usage.prompt_tokens if hasattr(usage, "prompt_tokens") else 0
-            )
+            prompt_tokens = usage.prompt_tokens if hasattr(usage, "prompt_tokens") else 0
             completion_tokens = (
                 usage.completion_tokens if hasattr(usage, "completion_tokens") else 0
             )
@@ -1020,28 +1108,29 @@ class FunctionWithUsage(Function):
             self.completion_tokens += completion_tokens
             self.total_tokens += total_tokens
         else:
-            if self.missing_usage_exception and not "preview" in kwargs:
-                raise Exception("Missing usage in metadata of neursymbolic engine")
-            else:
-                prompt_tokens = 0
-                completion_tokens = 0
-                total_tokens = 0
+            if self.missing_usage_exception and "preview" not in kwargs:
+                UserMessage(
+                    "Missing usage in metadata of neursymbolic engine", raise_with=Exception
+                )
+            prompt_tokens = 0
+            completion_tokens = 0
+            total_tokens = 0
 
         return res, self._format_usage(prompt_tokens, completion_tokens, total_tokens)
 
 
 class SelfPrompt(Expression):
-    _default_retry_tries     = 20
-    _default_retry_delay     = 0.5
+    _default_retry_tries = 20
+    _default_retry_delay = 0.5
     _default_retry_max_delay = -1
-    _default_retry_backoff   = 1
-    _default_retry_jitter    = 0
-    _default_retry_graceful  = True
+    _default_retry_backoff = 1
+    _default_retry_jitter = 0
+    _default_retry_graceful = True
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
-    def forward(self, existing_prompt: Dict[str, str], **kwargs) -> Dict[str, str]:
+    def forward(self, existing_prompt: dict[str, str], **kwargs) -> dict[str, str]:
         """
         Generate new system and user prompts based on the existing prompt.
 
@@ -1050,14 +1139,21 @@ class SelfPrompt(Expression):
         :return: A dictionary containing the new prompts in the same format:
                  {'user': '...', 'system': '...'}
         """
-        tries     = kwargs.get('tries', self._default_retry_tries)
-        delay     = kwargs.get('delay', self._default_retry_delay)
-        max_delay = kwargs.get('max_delay', self._default_retry_max_delay)
-        backoff   = kwargs.get('backoff', self._default_retry_backoff)
-        jitter    = kwargs.get('jitter', self._default_retry_jitter)
-        graceful  = kwargs.get('graceful', self._default_retry_graceful)
-
-        @core_ext.retry(tries=tries, delay=delay, max_delay=max_delay, backoff=backoff, jitter=jitter, graceful=graceful)
+        tries = kwargs.get("tries", self._default_retry_tries)
+        delay = kwargs.get("delay", self._default_retry_delay)
+        max_delay = kwargs.get("max_delay", self._default_retry_max_delay)
+        backoff = kwargs.get("backoff", self._default_retry_backoff)
+        jitter = kwargs.get("jitter", self._default_retry_jitter)
+        graceful = kwargs.get("graceful", self._default_retry_graceful)
+
+        @core_ext.retry(
+            tries=tries,
+            delay=delay,
+            max_delay=max_delay,
+            backoff=backoff,
+            jitter=jitter,
+            graceful=graceful,
+        )
         @core.zero_shot(
             prompt=(
                 "Based on the following prompt, generate a new system (or developer) prompt and a new user prompt. "
@@ -1066,18 +1162,19 @@ class SelfPrompt(Expression):
                 "The new user prompt should contain the user's requirements. "
                 "Check if the input contains a 'system' or 'developer' key and use the same key in your output. "
                 "Only output the new prompts in JSON format as shown:\n\n"
-                "{\"system\": \"<new system prompt>\", \"user\": \"<new user prompt>\"}\n\n"
+                '{"system": "<new system prompt>", "user": "<new user prompt>"}\n\n'
                 "OR\n\n"
-                "{\"developer\": \"<new developer prompt>\", \"user\": \"<new user prompt>\"}\n\n"
+                '{"developer": "<new developer prompt>", "user": "<new user prompt>"}\n\n'
                 "Maintain the same key structure as in the input prompt. Do not include any additional text."
             ),
             response_format={"type": "json_object"},
             post_processors=[
                 lambda res, _: json.loads(res),
             ],
-            **kwargs
+            **kwargs,
         )
-        def _func(self, sym: Symbol): pass
+        def _func(self, sym: Symbol):
+            pass
 
         return _func(self, self._to_symbol(existing_prompt))
 
@@ -1093,16 +1190,19 @@ class MetadataTracker(Expression):
     def __str__(self, value=None):
         value = value or self.metadata
         if isinstance(value, dict):
-            return '{\n\t' + ', \n\t'.join(f'"{k}": {self.__str__(v)}' for k,v in value.items()) + '\n}'
-        elif isinstance(value, list):
-            return '[' + ', '.join(self.__str__(item) for item in value) + ']'
-        elif isinstance(value, str):
+            return (
+                "{\n\t"
+                + ", \n\t".join(f'"{k}": {self.__str__(v)}' for k, v in value.items())
+                + "\n}"
+            )
+        if isinstance(value, list):
+            return "[" + ", ".join(self.__str__(item) for item in value) + "]"
+        if isinstance(value, str):
             return f'"{value}"'
-        else:
-            return f"\n\t    {value}"
+        return f"\n\t    {value}"
 
-    def __new__(cls, *args, **kwargs):
-        cls._lock = getattr(cls, '_lock', Lock())
+    def __new__(cls, *_args, **_kwargs):
+        cls._lock = getattr(cls, "_lock", Lock())
         with cls._lock:
             instance = super().__new__(cls)
             instance._metadata = {}
@@ -1122,25 +1222,26 @@ class MetadataTracker(Expression):
 
     def _trace_calls(self, frame, event, arg):
         if not self._trace:
-            return
+            return None
 
-        if event == 'return' and frame.f_code.co_name == 'forward':
-            # Check if this is an engine forward call
-            if ('self' in frame.f_locals
-                and
-                isinstance(frame.f_locals['self'], Engine)):
-                _, metadata = arg  # arg contains return value on 'return' event
-                engine_name = frame.f_locals['self'].__class__.__name__
-                model_name = frame.f_locals['self'].model
-                self._metadata[(self._metadata_id, engine_name, model_name)] = metadata
-                self._metadata_id += 1
+        if (
+            event == "return"
+            and frame.f_code.co_name == "forward"
+            and "self" in frame.f_locals
+            and isinstance(frame.f_locals["self"], Engine)
+        ):
+            _, metadata = arg  # arg contains return value on 'return' event
+            engine_name = frame.f_locals["self"].__class__.__name__
+            model_name = frame.f_locals["self"].model
+            self._metadata[(self._metadata_id, engine_name, model_name)] = metadata
+            self._metadata_id += 1
 
         return self._trace_calls
 
     def _accumulate_completion_token_details(self):
         """Parses the return object and accumulates completion token details per token type"""
         if not self._metadata:
-            CustomUserWarning("No metadata available to generate usage details.")
+            UserMessage("No metadata available to generate usage details.")
             return {}
 
         token_details = defaultdict(lambda: defaultdict(lambda: defaultdict(int)))
@@ -1151,46 +1252,149 @@ class MetadataTracker(Expression):
             try:
                 if engine_name == "GroqEngine":
                     usage = metadata["raw_output"].usage
-                    token_details[(engine_name, model_name)]["usage"]["completion_tokens"] += usage.completion_tokens
-                    token_details[(engine_name, model_name)]["usage"]["prompt_tokens"] += usage.prompt_tokens
-                    token_details[(engine_name, model_name)]["usage"]["total_tokens"] += usage.total_tokens
+                    token_details[(engine_name, model_name)]["usage"]["completion_tokens"] += (
+                        usage.completion_tokens
+                    )
+                    token_details[(engine_name, model_name)]["usage"]["prompt_tokens"] += (
+                        usage.prompt_tokens
+                    )
+                    token_details[(engine_name, model_name)]["usage"]["total_tokens"] += (
+                        usage.total_tokens
+                    )
                     token_details[(engine_name, model_name)]["usage"]["total_calls"] += 1
                     #!: Backward compatibility for components like `RuntimeInfo`
-                    token_details[(engine_name, model_name)]["prompt_breakdown"]["cached_tokens"] += 0 # Assignment not allowed with defualtdict
-                    token_details[(engine_name, model_name)]["completion_breakdown"]["reasoning_tokens"] += 0
+                    token_details[(engine_name, model_name)]["prompt_breakdown"][
+                        "cached_tokens"
+                    ] += 0  # Assignment not allowed with defualtdict
+                    token_details[(engine_name, model_name)]["completion_breakdown"][
+                        "reasoning_tokens"
+                    ] += 0
+                elif engine_name == "ParallelEngine":
+                    token_details[(engine_name, None)]["usage"]["total_calls"] += 1
+                    # There are no model-specific tokens for this engine
+                    token_details[(engine_name, None)]["usage"]["completion_tokens"] += 0
+                    token_details[(engine_name, None)]["usage"]["prompt_tokens"] += 0
+                    token_details[(engine_name, None)]["usage"]["total_tokens"] += 0
+                    #!: Backward compatibility for components like `RuntimeInfo`
+                    token_details[(engine_name, None)]["prompt_breakdown"]["cached_tokens"] += (
+                        0  # Assignment not allowed with defualtdict
+                    )
+                    token_details[(engine_name, None)]["completion_breakdown"][
+                        "reasoning_tokens"
+                    ] += 0
                 elif engine_name in ("GPTXChatEngine", "GPTXReasoningEngine"):
                     usage = metadata["raw_output"].usage
-                    token_details[(engine_name, model_name)]["usage"]["completion_tokens"] += usage.completion_tokens
-                    token_details[(engine_name, model_name)]["usage"]["prompt_tokens"] += usage.prompt_tokens
-                    token_details[(engine_name, model_name)]["usage"]["total_tokens"] += usage.total_tokens
+                    token_details[(engine_name, model_name)]["usage"]["completion_tokens"] += (
+                        usage.completion_tokens
+                    )
+                    token_details[(engine_name, model_name)]["usage"]["prompt_tokens"] += (
+                        usage.prompt_tokens
+                    )
+                    token_details[(engine_name, model_name)]["usage"]["total_tokens"] += (
+                        usage.total_tokens
+                    )
                     token_details[(engine_name, model_name)]["usage"]["total_calls"] += 1
-                    token_details[(engine_name, model_name)]["completion_breakdown"]["accepted_prediction_tokens"] += usage.completion_tokens_details.accepted_prediction_tokens
-                    token_details[(engine_name, model_name)]["completion_breakdown"]["rejected_prediction_tokens"] += usage.completion_tokens_details.rejected_prediction_tokens
-                    token_details[(engine_name, model_name)]["completion_breakdown"]["audio_tokens"] += usage.completion_tokens_details.audio_tokens
-                    token_details[(engine_name, model_name)]["completion_breakdown"]["reasoning_tokens"] += usage.completion_tokens_details.reasoning_tokens
-                    token_details[(engine_name, model_name)]["prompt_breakdown"]["audio_tokens"] += usage.prompt_tokens_details.audio_tokens
-                    token_details[(engine_name, model_name)]["prompt_breakdown"]["cached_tokens"] += usage.prompt_tokens_details.cached_tokens
+                    token_details[(engine_name, model_name)]["completion_breakdown"][
+                        "accepted_prediction_tokens"
+                    ] += usage.completion_tokens_details.accepted_prediction_tokens
+                    token_details[(engine_name, model_name)]["completion_breakdown"][
+                        "rejected_prediction_tokens"
+                    ] += usage.completion_tokens_details.rejected_prediction_tokens
+                    token_details[(engine_name, model_name)]["completion_breakdown"][
+                        "audio_tokens"
+                    ] += usage.completion_tokens_details.audio_tokens
+                    token_details[(engine_name, model_name)]["completion_breakdown"][
+                        "reasoning_tokens"
+                    ] += usage.completion_tokens_details.reasoning_tokens
+                    token_details[(engine_name, model_name)]["prompt_breakdown"][
+                        "audio_tokens"
+                    ] += usage.prompt_tokens_details.audio_tokens
+                    token_details[(engine_name, model_name)]["prompt_breakdown"][
+                        "cached_tokens"
+                    ] += usage.prompt_tokens_details.cached_tokens
                 elif engine_name == "GPTXSearchEngine":
                     usage = metadata["raw_output"].usage
-                    token_details[(engine_name, model_name)]["usage"]["prompt_tokens"] += usage.input_tokens
-                    token_details[(engine_name, model_name)]["usage"]["completion_tokens"] += usage.output_tokens
-                    token_details[(engine_name, model_name)]["usage"]["total_tokens"] += usage.total_tokens
+                    token_details[(engine_name, model_name)]["usage"]["prompt_tokens"] += (
+                        usage.input_tokens
+                    )
+                    token_details[(engine_name, model_name)]["usage"]["completion_tokens"] += (
+                        usage.output_tokens
+                    )
+                    token_details[(engine_name, model_name)]["usage"]["total_tokens"] += (
+                        usage.total_tokens
+                    )
                     token_details[(engine_name, model_name)]["usage"]["total_calls"] += 1
-                    token_details[(engine_name, model_name)]["prompt_breakdown"]["cached_tokens"] += usage.input_tokens_details.cached_tokens
-                    token_details[(engine_name, model_name)]["completion_breakdown"]["reasoning_tokens"] += usage.output_tokens_details.reasoning_tokens
+                    token_details[(engine_name, model_name)]["prompt_breakdown"][
+                        "cached_tokens"
+                    ] += usage.input_tokens_details.cached_tokens
+                    token_details[(engine_name, model_name)]["completion_breakdown"][
+                        "reasoning_tokens"
+                    ] += usage.output_tokens_details.reasoning_tokens
                 else:
                     logger.warning(f"Tracking {engine_name} is not supported.")
                     continue
             except Exception as e:
-                CustomUserWarning(f"Failed to parse metadata for {engine_name}: {e}", raise_with=AttributeError)
+                UserMessage(
+                    f"Failed to parse metadata for {engine_name}: {e}", raise_with=AttributeError
+                )
 
         # Convert to normal dict
         return {**token_details}
 
+    def _can_accumulate_engine(self, engine_name: str) -> bool:
+        supported_engines = ("GPTXChatEngine", "GPTXReasoningEngine", "GPTXSearchEngine")
+        return engine_name in supported_engines
+
+    def _accumulate_time_field(self, accumulated: dict, metadata: dict) -> None:
+        if "time" in metadata and "time" in accumulated:
+            accumulated["time"] += metadata["time"]
+
+    def _accumulate_usage_fields(self, accumulated: dict, metadata: dict) -> None:
+        if "raw_output" not in metadata or "raw_output" not in accumulated:
+            return
+
+        metadata_raw_output = metadata["raw_output"]
+        accumulated_raw_output = accumulated["raw_output"]
+        if not hasattr(metadata_raw_output, "usage") or not hasattr(
+            accumulated_raw_output, "usage"
+        ):
+            return
+
+        current_usage = metadata_raw_output.usage
+        accumulated_usage = accumulated_raw_output.usage
+
+        for attr in ["completion_tokens", "prompt_tokens", "total_tokens"]:
+            if hasattr(current_usage, attr) and hasattr(accumulated_usage, attr):
+                setattr(
+                    accumulated_usage,
+                    attr,
+                    getattr(accumulated_usage, attr) + getattr(current_usage, attr),
+                )
+
+        for detail_attr in ["completion_tokens_details", "prompt_tokens_details"]:
+            if not hasattr(current_usage, detail_attr) or not hasattr(
+                accumulated_usage, detail_attr
+            ):
+                continue
+
+            current_details = getattr(current_usage, detail_attr)
+            accumulated_details = getattr(accumulated_usage, detail_attr)
+
+            for attr in dir(current_details):
+                if attr.startswith("_") or not hasattr(accumulated_details, attr):
+                    continue
+
+                current_val = getattr(current_details, attr)
+                accumulated_val = getattr(accumulated_details, attr)
+                if isinstance(current_val, (int, float)) and isinstance(
+                    accumulated_val, (int, float)
+                ):
+                    setattr(accumulated_details, attr, accumulated_val + current_val)
+
     def _accumulate_metadata(self):
         """Accumulates metadata across all tracked engine calls."""
         if not self._metadata:
-            CustomUserWarning("No metadata available to generate usage details.")
+            UserMessage("No metadata available to generate usage details.")
             return {}
 
         # Use first entry as base
@@ -1199,39 +1403,14 @@ class MetadataTracker(Expression):
 
         # Skipz first entry
         for (_, engine_name), metadata in list(self._metadata.items())[1:]:
-            if engine_name not in ("GPTXChatEngine", "GPTXReasoningEngine", "GPTXSearchEngine"):
-                logger.warning(f"Metadata accumulation for {engine_name} is not supported. Try `.usage` instead for now.")
+            if not self._can_accumulate_engine(engine_name):
+                logger.warning(
+                    f"Metadata accumulation for {engine_name} is not supported. Try `.usage` instead for now."
+                )
                 continue
 
-            # Accumulate time if it exists
-            if 'time' in metadata and 'time' in accumulated:
-                accumulated['time'] += metadata['time']
-
-            # Handle usage stats accumulation
-            if 'raw_output' in metadata and 'raw_output' in accumulated:
-                if hasattr(metadata['raw_output'], 'usage') and hasattr(accumulated['raw_output'], 'usage'):
-                    current_usage = metadata['raw_output'].usage
-                    accumulated_usage = accumulated['raw_output'].usage
-
-                    # Accumulate token counts
-                    for attr in ['completion_tokens', 'prompt_tokens', 'total_tokens']:
-                        if hasattr(current_usage, attr) and hasattr(accumulated_usage, attr):
-                            setattr(accumulated_usage, attr,
-                                    getattr(accumulated_usage, attr) + getattr(current_usage, attr))
-
-                    # Handle nested token details if they exist
-                    for detail_attr in ['completion_tokens_details', 'prompt_tokens_details']:
-                        if hasattr(current_usage, detail_attr) and hasattr(accumulated_usage, detail_attr):
-                            current_details = getattr(current_usage, detail_attr)
-                            accumulated_details = getattr(accumulated_usage, detail_attr)
-
-                            # Accumulate all numeric attributes in the details
-                            for attr in dir(current_details):
-                                if not attr.startswith('_') and hasattr(accumulated_details, attr):
-                                    current_val = getattr(current_details, attr)
-                                    accumulated_val = getattr(accumulated_details, attr)
-                                    if isinstance(current_val, (int, float)) and isinstance(accumulated_val, (int, float)):
-                                        setattr(accumulated_details, attr, accumulated_val + current_val)
+            self._accumulate_time_field(accumulated, metadata)
+            self._accumulate_usage_fields(accumulated, metadata)
 
         return accumulated
 
@@ -1250,7 +1429,8 @@ class MetadataTracker(Expression):
 
 class DynamicEngine(Expression):
     """Context manager for dynamically switching neurosymbolic engine models."""
-    def __init__(self, model: str, api_key: str, debug: bool = False, **kwargs):
+
+    def __init__(self, model: str, api_key: str, _debug: bool = False, **_kwargs):
         super().__init__()
         self.model = model
         self.api_key = api_key
@@ -1259,8 +1439,8 @@ class DynamicEngine(Expression):
         self.engine_instance = None
         self._ctx_token = None
 
-    def __new__(cls, *args, **kwargs):
-        cls._lock = getattr(cls, '_lock', Lock())
+    def __new__(cls, *_args, **_kwargs):
+        cls._lock = getattr(cls, "_lock", Lock())
         with cls._lock:
             instance = super().__new__(cls)
             instance._metadata = {}
@@ -1293,11 +1473,178 @@ class DynamicEngine(Expression):
 
     def _create_engine_instance(self):
         """Create an engine instance based on the model name."""
-        from .backend.engines.neurosymbolic import ENGINE_MAPPING
+        # Deferred to avoid components <-> neurosymbolic engine circular imports.
+        from .backend.engines.neurosymbolic import ENGINE_MAPPING  # noqa
+
         try:
             engine_class = ENGINE_MAPPING.get(self.model)
             if engine_class is None:
-                raise ValueError(f"Unsupported model '{self.model}'")
+                UserMessage(f"Unsupported model '{self.model}'", raise_with=ValueError)
             return engine_class(api_key=self.api_key, model=self.model)
         except Exception as e:
-            raise ValueError(f"Failed to create engine for model '{self.model}': {str(e)}")
+            UserMessage(
+                f"Failed to create engine for model '{self.model}': {e!s}", raise_with=ValueError
+            )
+
+
+# Chonkie chunker imports - lazy loaded
+_CHONKIE_MODULES = None
+_CHUNKER_MAPPING = None
+_CHONKIE_AVAILABLE = None
+
+
+def _lazy_import_chonkie():
+    """Lazily import chonkie modules when needed."""
+    global _CHONKIE_MODULES, _CHUNKER_MAPPING, _CHONKIE_AVAILABLE
+
+    if _CHONKIE_MODULES is not None:
+        return _CHONKIE_MODULES
+
+    try:
+        from chonkie import (  # noqa
+            CodeChunker,
+            LateChunker,
+            NeuralChunker,
+            RecursiveChunker,
+            SemanticChunker,
+            SentenceChunker,
+            SlumberChunker,
+            TableChunker,
+            TokenChunker,
+        )
+        from chonkie.embeddings.base import BaseEmbeddings  # noqa
+        from tokenizers import Tokenizer  # noqa
+
+        _CHONKIE_MODULES = {
+            "CodeChunker": CodeChunker,
+            "LateChunker": LateChunker,
+            "NeuralChunker": NeuralChunker,
+            "RecursiveChunker": RecursiveChunker,
+            "SemanticChunker": SemanticChunker,
+            "SentenceChunker": SentenceChunker,
+            "SlumberChunker": SlumberChunker,
+            "TableChunker": TableChunker,
+            "TokenChunker": TokenChunker,
+            "BaseEmbeddings": BaseEmbeddings,
+            "Tokenizer": Tokenizer,
+        }
+        _CHUNKER_MAPPING = {
+            "TokenChunker": TokenChunker,
+            "SentenceChunker": SentenceChunker,
+            "RecursiveChunker": RecursiveChunker,
+            "SemanticChunker": SemanticChunker,
+            "CodeChunker": CodeChunker,
+            "LateChunker": LateChunker,
+            "NeuralChunker": NeuralChunker,
+            "SlumberChunker": SlumberChunker,
+            "TableChunker": TableChunker,
+        }
+        _CHONKIE_AVAILABLE = True
+    except ImportError:
+        _CHONKIE_MODULES = {}
+        _CHUNKER_MAPPING = {}
+        _CHONKIE_AVAILABLE = False
+
+    return _CHONKIE_MODULES
+
+
+def _get_chunker_mapping():
+    """Get the chunker mapping, lazily importing chonkie if needed."""
+    if _CHUNKER_MAPPING is None:
+        _lazy_import_chonkie()
+    return _CHUNKER_MAPPING or {}
+
+
+def _is_chonkie_available():
+    """Check if chonkie is available, lazily importing if needed."""
+    if _CHONKIE_AVAILABLE is None:
+        _lazy_import_chonkie()
+    return _CHONKIE_AVAILABLE or False
+
+
+@beartype
+class ChonkieChunker(Expression):
+    def __init__(
+        self,
+        tokenizer_name: str | None = "gpt2",
+        embedding_model_name: str | None = "minishlab/potion-base-8M",
+        **symai_kwargs,
+    ):
+        super().__init__(**symai_kwargs)
+        self.tokenizer_name = tokenizer_name
+        self.embedding_model_name = embedding_model_name
+
+    def forward(
+        self, data: Symbol, chunker_name: str | None = "RecursiveChunker", **chunker_kwargs
+    ) -> Symbol:
+        if not _is_chonkie_available():
+            UserMessage(
+                "chonkie library is not installed. Please install it with `pip install chonkie tokenizers`.",
+                raise_with=ImportError,
+            )
+        chunker = self._resolve_chunker(chunker_name, **chunker_kwargs)
+        chunks = [ChonkieChunker.clean_text(chunk.text) for chunk in chunker(data.value)]
+        return self._to_symbol(chunks)
+
+    def _resolve_chunker(self, chunker_name: str, **chunker_kwargs):
+        """Resolve and instantiate a chunker by name."""
+        chunker_mapping = _get_chunker_mapping()
+
+        if chunker_name not in chunker_mapping:
+            msg = (
+                f"Chunker {chunker_name} not found. Available chunkers: {list(chunker_mapping.keys())}. "
+                f"See docs (https://docs.chonkie.ai/getting-started/introduction) for more info."
+            )
+            raise ValueError(msg)
+
+        chunker_class = chunker_mapping[chunker_name]
+        chonkie_modules = _lazy_import_chonkie()
+        Tokenizer = chonkie_modules.get("Tokenizer")
+
+        # Tokenizer-based chunkers (use tokenizer_name)
+        if chunker_name in ["TokenChunker", "SentenceChunker", "RecursiveChunker"]:
+            if Tokenizer is None:
+                UserMessage(
+                    "Tokenizers library is not installed. Please install it with `pip install tokenizers`.",
+                    raise_with=ImportError,
+                )
+            tokenizer = Tokenizer.from_pretrained(self.tokenizer_name)
+            return chunker_class(tokenizer, **chunker_kwargs)
+
+        # Embedding-based chunkers (use embedding_model_name)
+        if chunker_name in ["SemanticChunker", "LateChunker"]:
+            return chunker_class(embedding_model=self.embedding_model_name, **chunker_kwargs)
+
+        # CodeChunker and TableChunker use tokenizer (can use string or Tokenizer object)
+        if chunker_name in ["CodeChunker", "TableChunker"]:
+            # These can accept tokenizer as string (default 'character') or Tokenizer object
+            # If tokenizer not provided in kwargs, use tokenizer_name
+            if "tokenizer" not in chunker_kwargs:
+                chunker_kwargs["tokenizer"] = self.tokenizer_name
+            return chunker_class(**chunker_kwargs)
+
+        # SlumberChunker uses tokenizer (can use string or Tokenizer object)
+        if chunker_name == "SlumberChunker":
+            # SlumberChunker can accept tokenizer as string or Tokenizer object
+            # If tokenizer not provided in kwargs, use tokenizer_name
+            if "tokenizer" not in chunker_kwargs:
+                chunker_kwargs["tokenizer"] = self.tokenizer_name
+            return chunker_class(**chunker_kwargs)
+
+        # NeuralChunker uses model parameter (defaults provided by chonkie)
+        if chunker_name == "NeuralChunker":
+            return chunker_class(**chunker_kwargs)
+
+        msg = (
+            f"Chunker {chunker_name} not properly configured. "
+            f"Available chunkers: {list(chunker_mapping.keys())}."
+        )
+        raise ValueError(msg)
+
+    @staticmethod
+    def clean_text(text: str) -> str:
+        """Cleans text by removing problematic characters."""
+        text = text.replace("\x00", "")  # Remove null bytes (\x00)
+        return text.encode("utf-8", errors="ignore").decode(
+            "utf-8"
+        )  # Replace invalid UTF-8 sequences