flock-core 0.4.513__tar.gz → 0.4.514__tar.gz

{flock_core-0.4.513 → flock_core-0.4.514}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flock-core
-Version: 0.4.513
+Version: 0.4.514
 Summary: Declarative LLM Orchestration at Scale
 Author-email: Andre Ratzenberger <andre.ratzenberger@whiteduck.de>
 License-File: LICENSE
@@ -69,6 +69,7 @@ Provides-Extra: all-tools
 Requires-Dist: azure-identity>=1.23.0; extra == 'all-tools'
 Requires-Dist: azure-search-documents>=11.5.2; extra == 'all-tools'
 Requires-Dist: azure-storage-blob>=12.25.1; extra == 'all-tools'
+Requires-Dist: docker>=7.1.0; extra == 'all-tools'
 Requires-Dist: docling>=2.18.0; extra == 'all-tools'
 Requires-Dist: duckduckgo-search>=7.3.2; extra == 'all-tools'
 Requires-Dist: markdownify>=0.14.1; extra == 'all-tools'
@@ -83,6 +84,8 @@ Requires-Dist: docling>=2.18.0; extra == 'basic-tools'
 Requires-Dist: duckduckgo-search>=7.3.2; extra == 'basic-tools'
 Requires-Dist: markdownify>=0.14.1; extra == 'basic-tools'
 Requires-Dist: tavily-python>=0.5.0; extra == 'basic-tools'
+Provides-Extra: code-tools
+Requires-Dist: docker>=7.1.0; extra == 'code-tools'
 Provides-Extra: evaluation
 Requires-Dist: datasets>=3.2.0; extra == 'evaluation'
 Requires-Dist: rouge-score>=0.1.2; extra == 'evaluation'

{flock_core-0.4.513 → flock_core-0.4.514}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "flock-core"
-version = "0.4.513"
+version = "0.4.514"
 description = "Declarative LLM Orchestration at Scale"
 readme = "README.md"
 authors = [
@@ -54,6 +54,7 @@ dependencies = [
     "opik>=1.7.26",
     "azure-data-tables>=12.7.0",
     "croniter>=6.0.0",
+    
 ]
 
 [project.optional-dependencies]
@@ -69,6 +70,7 @@ azure-tools = [
     "azure-search-documents>=11.5.2",
 ]
 llm-tools = ["nltk>=3.9.1"]
+code-tools = [ "docker>=7.1.0"]
 memory = [
     "zep-python>=2.0.2",
     "mem0ai[graph]>=0.1.101",
@@ -89,6 +91,7 @@ all-tools = [
     "azure-storage-blob>=12.25.1",
     "azure-search-documents>=11.5.2",
     "nltk>=3.9.1",
+    "docker>=7.1.0",
 ]
 all = [
     "docling>=2.18.0",

flock_core-0.4.514/src/flock/core/execution/opik_executor.py

@@ -0,0 +1,103 @@
+# src/flock/core/execution/evaluation_processor.py
+"""Contains the EvaluationProcessor class responsible for evaluating Flock agents
+against datasets using various metrics.
+"""
+
+from pathlib import Path
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Union,
+)
+
+from opik import Opik
+from pandas import DataFrame
+
+# Conditional pandas import
+try:
+    import pandas as pd
+
+    PANDAS_AVAILABLE = True
+except ImportError:
+    pd = None  # type: ignore
+    PANDAS_AVAILABLE = False
+
+# Box for results
+from datasets import Dataset as HFDataset
+
+from flock.core.evaluation.utils import (
+    normalize_dataset,
+    # Import metric calculation/aggregation helpers
+)
+
+# Flock core imports
+from flock.core.logging.logging import get_logger
+
+if TYPE_CHECKING:
+    from flock.core.flock import Flock
+    from flock.core.flock_agent import FlockAgent
+    # Conditional types
+
+
+logger = get_logger("execution.opik")
+
+
+class OpikExecutor:
+    """Handles the evaluation of Flock agents against datasets."""
+
+    def __init__(self, flock_instance: "Flock"):
+        """Initializes the EvaluationProcessor.
+
+        Args:
+            flock_instance: The Flock instance this processor will use.
+        """
+        self.flock = flock_instance
+
+    async def evaluate_with_opik(
+        self,
+        dataset: str | Path | list[dict[str, Any]] | DataFrame | HFDataset,
+        start_agent: Union["FlockAgent", str],
+        input_mapping: dict[str, str],
+        answer_mapping: dict[str, str],) -> DataFrame | list[dict[str, Any]]:
+        """Evaluates the Flock's performance against a dataset asynchronously."""
+        logger.info(f"Evaluating Flock's performance against dataset: {dataset}")
+
+        # Evaluation task
+        def evaluation_task(dataset_item):
+          flock_result = self.flock.run(start_agent=start_agent, input=dataset_item, box_result=False)
+
+          result = {
+              "input": dataset_item.get("test"),
+              "output": flock_result.get("answer"),
+              "context": ["placeholder string"]
+          }
+
+          return result
+
+        start_agent_name = (
+            start_agent.name if hasattr(start_agent, "name") else start_agent
+        )
+        dataset_name = str(dataset)
+
+        # --- 1. Normalize Dataset ---
+        try:
+            df = normalize_dataset(dataset)  # Uses helper
+            if df is None or df.empty:
+                raise ValueError(
+                    "Provided dataset is empty or could not be normalized."
+                )
+            logger.info(f"Normalized dataset with {len(df)} items.")
+        except Exception as e:
+            logger.error(
+                f"Failed to load or normalize dataset: {e}", exc_info=True
+            )
+            raise ValueError(f"Dataset processing failed: {e}") from e
+
+        logger.info(f"type(df): {type(df)}")        # ➜ <class 'pandas.core.frame.DataFrame'>
+        logger.info(f"df.shape: {df.shape}")        # e.g. (123456, N_COLUMNS+2)
+        logger.info(f"df['split'].value_counts(): {df['split'].value_counts()}")
+        logger.info(f"df['config'].unique(): {df['config'].unique()}")
+        client = Opik()
+        dataset = client.get_or_create_dataset(name=dataset_name)
+        dataset.insert_from_pandas(dataframe=df, ignore_keys=["source"])
+        logger.info(f"Imported dataset to Opik")

{flock_core-0.4.513 → flock_core-0.4.514}/src/flock/core/interpreter/python_interpreter.py

@@ -37,6 +37,14 @@ class InterpreterError(ValueError):
 
     pass
 
+class BreakException(Exception):
+    """Signal a 'break' from the simulated loop."""
+    pass
+
+class ContinueException(Exception):
+    """Signal a 'continue' in the simulated loop."""
+    pass
+
 
 class PythonInterpreter:
     r"""A customized python interpreter to control the execution of
@@ -44,8 +52,6 @@ class PythonInterpreter:
     functions given in action space and import white list. It also supports
     fuzzy variable matching to receive uncertain input variable name.
 
-    [Documentation omitted for brevity]
-
     Args:
         action_space (Dict[str, Any]): A dictionary mapping action names to
             their corresponding functions or objects.
@@ -76,6 +82,9 @@ class PythonInterpreter:
             "enum",
             "json",
             "ast",
+            "numpy",
+            "sympy",
+            "pandas",
         ]  # default imports
         self.verbose = verbose
 
@@ -95,50 +104,57 @@ class PythonInterpreter:
 
         [Documentation omitted for brevity]
         """
-        if state is not None:
-            self.state.update(state)
-        if fuzz_state is not None:
-            self.fuzz_state.update(fuzz_state)
-
         try:
-            expression = ast.parse(code)
-        except SyntaxError as e:
-            error_line = code.splitlines()[e.lineno - 1]
-            raise InterpreterError(
-                f"Syntax error in code at line {e.lineno}: {error_line}\nError: {e}"
-            )
+            if state is not None:
+                self.state.update(state)
+            if fuzz_state is not None:
+                self.fuzz_state.update(fuzz_state)
 
-        result = None
-        if self.verbose:
-            self.log("[Interpreter] Starting code execution...")
+            try:
+                expression = ast.parse(code)
+            except SyntaxError as e:
+                error_line = code.splitlines()[e.lineno - 1]
+                self.log(
+                    f"[Interpreter] Syntax error in code at line {e.lineno}: {error_line}\nError: {e}")
+                return(
+                    f"Syntax error in code at line {e.lineno}: {error_line}\nError: {e}"
+                )
 
-        for idx, node in enumerate(expression.body):
-            # Log the AST node being executed (using unparse if available)
+            result = None
             if self.verbose:
-                try:
-                    node_repr = ast.unparse(node)
-                except Exception:
-                    node_repr = ast.dump(node)
-                self.log(f"[Interpreter] Executing node {idx}: {node_repr}")
+                self.log("[Interpreter] Starting code execution...")
 
-            try:
-                line_result = self._execute_ast(node)
-            except InterpreterError as e:
-                if not keep_state:
-                    self.clear_state()
-                msg = f"Evaluation of the code stopped at node {idx}. See:\n{e}"
-                raise InterpreterError(msg)
-            if line_result is not None:
-                result = line_result
+            for idx, node in enumerate(expression.body):
+                # Log the AST node being executed (using unparse if available)
                 if self.verbose:
-                    self.log(f"[Interpreter] Node {idx} result: {result}")
+                    try:
+                        node_repr = ast.unparse(node)
+                    except Exception:
+                        node_repr = ast.dump(node)
+                    self.log(f"[Interpreter] Executing node {idx}: {node_repr}")
 
-        if self.verbose:
-            self.log("[Interpreter] Finished code execution.")
-        if not keep_state:
-            self.clear_state()
+                try:
+                    line_result = self._execute_ast(node)
+                except InterpreterError as e:
+                    if not keep_state:
+                        self.clear_state()
+                    msg = f"Evaluation of the code stopped at node {idx}. See:\n{e}"
+                    return msg
+                if line_result is not None:
+                    result = line_result
+                    if self.verbose:
+                        self.log(f"[Interpreter] Node {idx} result: {result}")
 
-        return result
+            if self.verbose:
+                self.log("[Interpreter] Finished code execution.")
+            if not keep_state:
+                self.clear_state()
+
+            return result
+        except Exception as e:
+            self.log(
+                f"[Interpreter] Error during code execution: {e}")
+            return f"[Interpreter] Error during code execution: {e}"
 
     def clear_state(self) -> None:
         r"""Initialize :obj:`state` and :obj:`fuzz_state`"""
@@ -236,7 +252,7 @@ class PythonInterpreter:
         elif isinstance(expression, ast.Assert):
             return self._execute_assert(expression)
         else:
-            raise InterpreterError(
+            return(
                 f"{expression.__class__.__name__} is not supported."
             )
 
@@ -253,17 +269,17 @@ class PythonInterpreter:
             self.state[target.id] = value
         elif isinstance(target, ast.Tuple):
             if not isinstance(value, tuple):
-                raise InterpreterError(
+                return(
                     f"Expected type tuple, but got {value.__class__.__name__} instead."
                 )
             if len(target.elts) != len(value):
-                raise InterpreterError(
+                return(
                     f"Expected {len(target.elts)} values but got {len(value)}."
                 )
             for t, v in zip(target.elts, value):
                 self.state[self._execute_ast(t)] = v
         else:
-            raise InterpreterError(
+            return(
                 f"Unsupported variable type. Expected ast.Name or ast.Tuple, got {target.__class__.__name__} instead."
             )
 
@@ -297,7 +313,7 @@ class PythonInterpreter:
             isinstance(current_value, (int, float))
             and isinstance(increment_value, (int, float))
         ):
-            raise InterpreterError(
+            return(
                 f"Invalid types for augmented assignment: {type(current_value)}, {type(increment_value)}"
             )
         if isinstance(augassign.op, ast.Add):
@@ -309,7 +325,7 @@ class PythonInterpreter:
         elif isinstance(augassign.op, ast.Div):
             new_value = current_value / increment_value
         else:
-            raise InterpreterError(
+            return(
                 f"Augmented assignment operator {augassign.op} is not supported"
             )
         self._assign(augassign.target, new_value)
@@ -319,7 +335,7 @@ class PythonInterpreter:
         index = self._execute_ast(subscript.slice)
         value = self._execute_ast(subscript.value)
         if not isinstance(subscript.ctx, ast.Load):
-            raise InterpreterError(
+            return(
                 f"{subscript.ctx.__class__.__name__} is not supported for subscript."
             )
         if isinstance(value, (list, tuple)):
@@ -330,7 +346,7 @@ class PythonInterpreter:
             close_matches = difflib.get_close_matches(index, list(value.keys()))
             if len(close_matches) > 0:
                 return value[close_matches[0]]
-        raise InterpreterError(f"Could not index {value} with '{index}'.")
+        return(f"Could not index {value} with '{index}'.")
 
     def _execute_name(self, name: ast.Name):
         if name.id in dir(builtins):
@@ -340,7 +356,7 @@ class PythonInterpreter:
         elif isinstance(name.ctx, ast.Load):
             return self._get_value_from_state(name.id)
         else:
-            raise InterpreterError(f"{name.ctx} is not supported.")
+            return(f"{name.ctx} is not supported.")
 
     def _execute_condition(self, condition):
         if isinstance(condition, ast.BoolOp):
@@ -355,12 +371,12 @@ class PythonInterpreter:
                 ]
                 return any(results)
             else:
-                raise InterpreterError(
+                return(
                     f"Boolean operator {condition.op} is not supported"
                 )
         elif isinstance(condition, ast.Compare):
             if len(condition.ops) > 1:
-                raise InterpreterError(
+                return(
                     "Cannot evaluate conditions with multiple operators"
                 )
             left = self._execute_ast(condition.left)
@@ -387,7 +403,7 @@ class PythonInterpreter:
             elif isinstance(comparator, ast.NotIn):
                 return left not in right
             else:
-                raise InterpreterError("Unsupported comparison operator")
+                return("Unsupported comparison operator")
         elif isinstance(condition, ast.UnaryOp):
             return self._execute_unaryop(condition)
         elif isinstance(condition, ast.Name) or isinstance(condition, ast.Call):
@@ -395,7 +411,7 @@ class PythonInterpreter:
         elif isinstance(condition, ast.Constant):
             return bool(condition.value)
         else:
-            raise InterpreterError(
+            return(
                 f"Unsupported condition type: {type(condition).__name__}"
             )
 
@@ -428,7 +444,7 @@ class PythonInterpreter:
 
     def _execute_import_from(self, import_from: ast.ImportFrom):
         if import_from.module is None:
-            raise InterpreterError('"from . import" is not supported.')
+            return('"from . import" is not supported.')
         for import_name in import_from.names:
             full_name = import_from.module + f".{import_name.name}"
             self._validate_import(full_name)
@@ -440,12 +456,6 @@ class PythonInterpreter:
     # We keep both as provided, but you may wish to consolidate these in your code.
 
     def _execute_for(self, for_statement: ast.For):
-        class BreakException(Exception):
-            pass
-
-        class ContinueException(Exception):
-            pass
-
         result = None
         try:
             for value in self._execute_ast(for_statement.iter):
@@ -462,12 +472,6 @@ class PythonInterpreter:
         return result
 
     def _execute_while(self, while_statement: ast.While):
-        class BreakException(Exception):
-            pass
-
-        class ContinueException(Exception):
-            pass
-
         result = None
         try:
             while self._execute_condition(while_statement.test):
@@ -540,7 +544,7 @@ class PythonInterpreter:
                 found_name = True
                 return
         if not found_name:
-            raise InterpreterError(
+            return(
                 f"It is not permitted to import modules "
                 f"than module white list (try to import {full_name})."
             )
@@ -577,7 +581,7 @@ class PythonInterpreter:
         elif isinstance(operator, ast.MatMult):
             return left @ right
         else:
-            raise InterpreterError(f"Operator not supported: {operator}")
+            return(f"Operator not supported: {operator}")
 
     def _execute_unaryop(self, unaryop: ast.UnaryOp):
         operand = self._execute_ast(unaryop.operand)
@@ -592,31 +596,33 @@ class PythonInterpreter:
         elif isinstance(operator, ast.Invert):
             return ~operand
         else:
-            raise InterpreterError(f"Operator not supported: {operator}")
+            return(f"Operator not supported: {operator}")
 
     def _execute_listcomp(self, comp: ast.ListComp):
-        return [self._execute_comp(comp.elt, comp.generators)]
-
-    def _execute_dictcomp(self, comp: ast.DictComp):
-        return {
-            self._execute_comp(comp.key, comp.generators): self._execute_comp(
-                comp.value, comp.generators
-            )
-        }
+        return self._execute_comp(comp.elt, comp.generators)
 
     def _execute_setcomp(self, comp: ast.SetComp):
-        return {self._execute_comp(comp.elt, comp.generators)}
+        return set(self._execute_comp(comp.elt, comp.generators))
+
+    def _execute_dictcomp(self, comp: ast.DictComp):
+        keys   = self._execute_comp(comp.key,   comp.generators)
+        values = self._execute_comp(comp.value, comp.generators)
+        return dict(zip(keys, values))
 
     def _execute_comp(self, elt, generators):
+        # Base-case: wrap the single element in a list so that
+        # callers can safely .extend() it.
         if not generators:
-            return self._execute_ast(elt)
+            return [self._execute_ast(elt)]
+
         gen = generators[0]
-        result = []
+        acc: list[Any] = []
         for value in self._execute_ast(gen.iter):
             self._assign(gen.target, value)
             if all(self._execute_condition(if_cond) for if_cond in gen.ifs):
-                result.extend(self._execute_comp(elt, generators[1:]))
-        return result
+                acc.extend(self._execute_comp(elt, generators[1:]))
+        return acc
+
 
     def _execute_generatorexp(self, genexp: ast.GeneratorExp):
         def generator():
@@ -631,7 +637,7 @@ class PythonInterpreter:
         elif key in self.fuzz_state:
             return self.fuzz_state[key]
         else:
-            raise InterpreterError(f"The variable `{key}` is not defined.")
+            return(f"The variable `{key}` is not defined.")
 
 
 class TextPrompt(str):

{flock_core-0.4.513 → flock_core-0.4.514}/src/flock/core/mixin/dspy_integration.py

@@ -9,7 +9,7 @@ from typing import Any, Literal
 from dspy import Tool
 
 from flock.core.logging.logging import get_logger
-from flock.core.util.spliter import split_top_level
+from flock.core.util.splitter import split_top_level
 
 # Import split_top_level (assuming it's moved or copied appropriately)
 # Option 1: If moved to a shared util

{flock_core-0.4.513 → flock_core-0.4.514}/src/flock/core/util/input_resolver.py

@@ -1,7 +1,7 @@
 """Utility functions for resolving input keys to their corresponding values."""
 
 from flock.core.context.context import FlockContext
-from flock.core.util.spliter import split_top_level
+from flock.core.util.splitter import split_top_level
 
 
 def get_callable_members(obj):

flock_core-0.4.514/src/flock/tools/code_tools.py

@@ -0,0 +1,167 @@
+from flock.core.interpreter.python_interpreter import PythonInterpreter
+from flock.core.logging.trace_and_logged import traced_and_logged
+
+
+@traced_and_logged
+def code_evaluate_math(expression: str) -> float:
+    try:
+        result = PythonInterpreter(
+            {},
+            [
+                "os",
+                "math",
+                "random",
+                "datetime",
+                "time",
+                "string",
+                "collections",
+                "itertools",
+                "functools",
+                "typing",
+                "enum",
+                "json",
+                "ast",
+            ],
+            verbose=True,
+        ).execute(expression)
+        return result
+    except Exception:
+        raise
+
+
+@traced_and_logged
+def code_code_eval(python_code: str) -> str:
+    """A Python code evaluation tool that executes Python code and returns the result.
+    
+    The code may not be markdown-escaped with triple backticks.
+    It is expected to be a valid Python code snippet that can be executed directly.
+    The code is executed in a controlled environment with a limited set of libraries.
+    It allows the use of the following libraries:
+                "os",
+                "math",
+                "random",
+                "datetime",
+                "time",
+                "string",
+                "collections",
+                "itertools",
+                "functools",
+                "typing",
+                "enum",
+                "json",
+                "ast",
+                "numpy",
+                "sympy",
+                "pandas",
+                "httpx",
+    """
+    try:
+        result = PythonInterpreter(
+            {},
+            [
+                "os",
+                "math",
+                "random",
+                "datetime",
+                "time",
+                "string",
+                "collections",
+                "itertools",
+                "functools",
+                "typing",
+                "enum",
+                "json",
+                "ast",
+                "numpy",
+                "sympy",
+                "pandas",
+            ],
+            verbose=True,
+        ).execute(python_code)
+        return result
+    except Exception:
+        raise
+
+
+@traced_and_logged
+def docker_code_execute(python_code: str) -> str:
+    """Execute Python code in a sandboxed Docker container."""
+    import ast
+    import os
+    import pathlib
+    import platform
+    import shutil
+    import textwrap
+    import uuid
+
+    import docker
+    def _auto_print_last_expr(code: str) -> str:
+        """If the last top-level statement is a bare expression,
+        append a print() so script mode surfaces its value.
+        """
+        tree = ast.parse(code, mode="exec")
+        if tree.body and isinstance(tree.body[-1], ast.Expr):
+            # Re-extract the exact source of that expression
+            expr_src = textwrap.dedent(
+                code.splitlines()[tree.body[-1].lineno - 1]
+            )
+            code += f"\nprint({expr_src})"
+        return code
+    # --- 1. Figure out a base directory that exists on this OS ----------
+    if platform.system() == "Windows":
+        base_dir = pathlib.Path(os.getenv("SANDBOX_BASE_DIR", r"C:\sandboxes"))
+    else:  # Linux, macOS, WSL2
+        base_dir = pathlib.Path(os.getenv("SANDBOX_BASE_DIR", "/var/sandboxes"))
+
+    base_dir.mkdir(parents=True, exist_ok=True)
+
+    sandbox_id = f"sbox-{uuid.uuid4()}"
+    workdir = base_dir / sandbox_id
+    workdir.mkdir(parents=True, exist_ok=False)
+
+    # Docker’s HTTP API always wants POSIX‐style paths (“/”, drive letter allowed).
+    host_path = workdir.resolve().as_posix()        # e.g. "C:/sandboxes/…"
+
+    client = docker.from_env()
+    image = "python:3.12-slim"
+
+    # --- 2. Decide whether we can / should request the gVisor runtime ---
+    runtime_args = {}
+    if platform.system() != "Windows" and shutil.which("runsc"):
+        runtime_args["runtime"] = "runsc"           # gVisor on Linux & macOS
+
+    container = client.containers.run(
+        image,
+        name=sandbox_id,
+        command=["sleep", "infinity"],
+        user="65534:65534",                # nobody
+        network_mode="none",
+        volumes={host_path: {"bind": "/workspace", "mode": "rw"}},
+        mem_limit="4g",
+        cpu_period=100_000,
+        cpu_quota=200_000,                 # 2 vCPU
+        security_opt=["no-new-privileges"],
+        detach=True,
+        **runtime_args,
+    )
+
+    try:
+        def exec_code(cmd: list[str], timeout: int = 30) -> str:
+            exec_id = client.api.exec_create(
+                container.id, cmd, workdir="/workspace"
+            )["Id"]
+            return client.api.exec_start(
+                exec_id, stream=False, demux=False, tty=False,
+            ).decode()
+
+        # --- 3. Copy code in and execute --------------------------------
+        (workdir / "main.py").write_text(_auto_print_last_expr(python_code), encoding="utf-8")
+        stdout = exec_code(["python", "main.py"], timeout=30)
+        return stdout.strip()
+
+    finally:
+        # --- 4. Tear everything down ------------------------------------
+        container.remove(force=True)
+        shutil.rmtree(workdir, ignore_errors=True)
+
+