ape-framework 0.1.2__tar.gz → 2.0.0.dev1__tar.gz

ape_framework-2.0.0.dev1/LICENSE

@@ -0,0 +1,14 @@
+APE (Algebra Problems Evaluator) is a framework for building and running benchmarks that evaluate LLMs on their ability to solve algebra problems.
+Copyright (C) 2025-2026 Adrian Boguszewski, Kacper Grzybowski, Maciej Teterycz, Mateusz Kasprzak
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <https://www.gnu.org/licenses/>.

{ape_framework-0.1.2 → ape_framework-2.0.0.dev1}/PKG-INFO

@@ -1,12 +1,35 @@
 Metadata-Version: 2.4
 Name: ape-framework
-Version: 0.1.2
+Version: 2.0.0.dev1
 Summary: Package for evaluating algebra problems using AI systems.
+License: APE (Algebra Problems Evaluator) is a framework for building and running benchmarks that evaluate LLMs on their ability to solve algebra problems.
+        Copyright (C) 2025-2026 Adrian Boguszewski, Kacper Grzybowski, Maciej Teterycz, Mateusz Kasprzak
+        
+        This program is free software: you can redistribute it and/or modify
+        it under the terms of the GNU General Public License as published by
+        the Free Software Foundation, either version 3 of the License, or any later version.
+        
+        This program is distributed in the hope that it will be useful,
+        but WITHOUT ANY WARRANTY; without even the implied warranty of
+        MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+        GNU General Public License for more details.
+        
+        You should have received a copy of the GNU General Public License
+        along with this program. If not, see <https://www.gnu.org/licenses/>.
 Classifier: Programming Language :: Python :: 3
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: langchain
+Requires-Dist: langgraph
+Requires-Dist: rich
+Requires-Dist: jsonpickle
+Requires-Dist: docker
+Requires-Dist: psycopg[binary,pool]
+Requires-Dist: pglast
+Requires-Dist: tiktoken
+Requires-Dist: pydantic
 Dynamic: license-file
 
 # Algebra Problems Evaluator (APE)

ape_framework-2.0.0.dev1/ape/__init__.py

@@ -0,0 +1,50 @@
+import logging
+
+from .adapter import SolutionAdapter
+from .checker import Checker, CheckerFactory, CheckResult
+from .cli import CLI
+from .llm import EntryKind, Session, Solver, SolverConfig
+from .report import Report
+from .run import run
+from .runner import (
+    CheckerState,
+    CheckerStatus,
+    NullObserver,
+    OutputBatch,
+    RunObserver,
+    RunParams,
+    SolverState,
+    SolverStatus,
+    WorkerSnapshot,
+)
+from .runner.runner import Runner
+from .visualizer import Visualizer
+
+# Prevent logging if the user did not configure it
+logger = logging.getLogger(__name__)
+logger.addHandler(logging.NullHandler())
+
+__all__ = [
+    "SolutionAdapter",
+    "Checker",
+    "CheckerFactory",
+    "CheckResult",
+    "CLI",
+    "Report",
+    "Visualizer",
+    "Runner",
+    "RunParams",
+    "Solver",
+    "SolverConfig",
+    "OutputBatch",
+    "RunObserver",
+    "NullObserver",
+    "WorkerSnapshot",
+    "SolverState",
+    "CheckerState",
+    "SolverStatus",
+    "CheckerStatus",
+    "Session",
+    "EntryKind",
+    "run",
+]

ape_framework-2.0.0.dev1/ape/__main__.py

@@ -0,0 +1,10 @@
+"""Allow ``python -m ape <config.toml>`` as the package entry point.
+
+This avoids the ``RuntimeWarning`` that ``python -m ape.run`` triggers (because
+``ape/__init__.py`` imports ``ape.run``); ``python -m ape.run`` still works.
+"""
+
+from .run import main
+
+if __name__ == "__main__":
+    main()

ape_framework-2.0.0.dev1/ape/adapter.py

@@ -0,0 +1,3 @@
+from typing import Callable
+
+type SolutionAdapter[SolutionT] = Callable[[str], SolutionT]

ape_framework-2.0.0.dev1/ape/checker/__init__.py

@@ -0,0 +1,11 @@
+from .checker import Checker, CheckerFactory, CheckResult, Failed, Passed
+from .orchestrator import CheckerOrchestrator
+
+__all__ = [
+    "Checker",
+    "CheckerFactory",
+    "CheckResult",
+    "Failed",
+    "Passed",
+    "CheckerOrchestrator",
+]

ape_framework-2.0.0.dev1/ape/checker/checker.py

@@ -0,0 +1,32 @@
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from datetime import timedelta
+from typing import Any, Callable, Optional
+
+from ..data import DataModel
+
+
+@dataclass(frozen=True)
+class Passed:
+    runtime: Optional[timedelta] = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(frozen=True)
+class Failed:
+    reason: str
+    runtime: Optional[timedelta] = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+type CheckResult = Passed | Failed
+
+
+class Checker[DataT: DataModel, SolutionT](ABC):
+    @abstractmethod
+    def check(self, data: DataT, solution: SolutionT) -> CheckResult: ...
+
+
+type CheckerFactory[DataT: DataModel, SolutionT] = Callable[
+    [], Checker[DataT, SolutionT]
+]

ape_framework-2.0.0.dev1/ape/checker/orchestrator.py

@@ -0,0 +1,92 @@
+import logging
+from concurrent.futures.thread import ThreadPoolExecutor
+from queue import Queue
+from threading import Event
+
+from ..data import DataModel
+from ..runner.monitor import (
+    CheckerState,
+    CheckerStatus,
+    WorkerStatusTracker,
+)
+from .checker import Checker, CheckerFactory, CheckResult
+
+logger = logging.getLogger(__name__)
+
+
+class CheckerOrchestrator[DataT: DataModel, SolutionT]:
+
+    def __init__(
+        self,
+        checker_factory: CheckerFactory[DataT, SolutionT],
+        checker_workers: int,
+        tracker: WorkerStatusTracker,
+        retry_timeout: int = 10,
+    ) -> None:
+        self._tracker = tracker
+        self._retry_timeout = retry_timeout
+
+        self._cancel = Event()
+        self._executor = ThreadPoolExecutor(max_workers=checker_workers)
+        self._checker_carousel: Queue[tuple[Checker[DataT, SolutionT], int]] = Queue()
+        for checker_id in range(checker_workers):
+            self._checker_carousel.put((checker_factory(), checker_id))
+
+    def check(self, data: DataT, solution: SolutionT, solver_id: int) -> CheckResult:
+        # This check is TOCTOU, but it doesn't matter - best effort to not put any new futures into the pool
+        if self._cancel.is_set():
+            raise InterruptedError
+
+        fut = self._executor.submit(self._checker_worker, data, solution, solver_id)
+        while True:
+            try:
+                check_result: CheckResult = fut.result(timeout=self._retry_timeout)
+                return check_result
+            except TimeoutError:
+                if self._cancel.is_set():
+                    raise InterruptedError
+                continue
+            except Exception:
+                logger.exception(
+                    "Checker orchestrator caught an exception from checker worker, propagating."
+                )
+                raise
+        raise RuntimeError("unreachable")  # Make PyCharm happy
+
+    def stop(self) -> None:
+        self._cancel.set()
+        self._executor.shutdown(cancel_futures=True)
+
+    def _checker_worker(
+        self, data: DataT, solution: SolutionT, solver_id: int
+    ) -> CheckResult:
+        checker, checker_id = self._checker_carousel.get()
+
+        logger.debug("Checker worker started, got checker with id %d.", checker_id)
+        self._tracker.update_checker(
+            CheckerStatus(
+                id=checker_id,
+                state=CheckerState.BUSY,
+                data_id=data.data_id,
+                solver_id=solver_id,
+            )
+        )
+
+        try:
+            check_result: CheckResult = checker.check(data, solution)
+        except Exception:
+            logger.exception("Checker worker encountered an unexpected exception.")
+            raise
+        finally:
+            self._tracker.update_checker(
+                CheckerStatus(
+                    id=checker_id,
+                    state=CheckerState.IDLE,
+                    data_id=None,
+                    solver_id=None,
+                )
+            )
+            self._checker_carousel.put((checker, checker_id))
+
+        logger.debug("Checker worker for checker with id %d has finished.", checker_id)
+        return check_result

{ape_framework-0.1.2 → ape_framework-2.0.0.dev1}/ape/cli.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import logging
-import logging.config
 from collections import deque
 from contextlib import AbstractContextManager
 from datetime import datetime
@@ -10,7 +9,7 @@ from types import TracebackType
 from typing import Any, Callable, Literal, Optional
 
 from rich import box
-from rich.console import Console
+from rich.console import Console, ConsoleOptions, RenderResult
 from rich.layout import Layout
 from rich.live import Live
 from rich.panel import Panel
@@ -24,8 +23,22 @@ from rich.progress import (
     TimeElapsedColumn,
 )
 from rich.table import Table
+from rich.text import Text
 
-from .runner import Runner
+from .llm.session_log import configure_session_log_file
+from .runner.monitor import CheckerStatus, SolverStatus, WorkerSnapshot
+
+
+class _DashboardRenderable:
+    """Thin wrapper so Live's refresh thread calls _render_dashboard each cycle."""
+
+    def __init__(self, cli: CLI) -> None:
+        self._cli = cli
+
+    def __rich_console__(
+        self, console: Console, options: ConsoleOptions
+    ) -> RenderResult:
+        yield self._cli._render_dashboard()
 
 
 class LogBufferHandler(logging.Handler):
@@ -35,10 +48,12 @@ class LogBufferHandler(logging.Handler):
 
     def emit(self, record: logging.LogRecord) -> None:
         try:
+            # Show only the last segment of the logger name to save space
+            short_name = record.name.rsplit(".", 1)[-1]
             entry = {
                 "time": datetime.fromtimestamp(record.created).strftime("%H:%M:%S"),
                 "level": record.levelname,
-                "logger": record.name,
+                "logger": short_name,
                 "message": record.getMessage(),
             }
             self._add_entry(entry)
@@ -47,20 +62,27 @@ class LogBufferHandler(logging.Handler):
 
 
 class CLI(AbstractContextManager):
+    """Rich-based dashboard. Implements ``RunObserver`` — pass the CLI
+    instance straight to ``Runner.run(observer=...)``::
+
+        with CLI(...) as cli:
+            Runner.run(..., observer=cli)
+    """
+
     def __init__(
         self,
-        runner: Runner,
         *,
         log_file: Path | str = Path("output/run.log"),
+        session_log_file: Path | str = Path("output/sessions.log"),
         log_level: int | str = logging.INFO,
     ) -> None:
-        self._runner = runner
         self._log_file = Path(log_file)
+        self._session_log_file = Path(session_log_file)
         self._log_level = self._resolve_log_level(log_level)
         self._progress_task_id: Optional[TaskID] = None
         self._log_buffer: deque[dict[str, str]] = deque(maxlen=50)
-        self._solver_statuses: list[dict[str, Any]] = []
-        self._checker_statuses: list[dict[str, Any]] = []
+        self._solver_statuses: list[SolverStatus] = []
+        self._checker_statuses: list[CheckerStatus] = []
 
     def __enter__(self) -> CLI:
         self._setup_logging()
@@ -81,14 +103,13 @@ class CLI(AbstractContextManager):
         self._log_handler.setLevel(self._log_level)
         logging.getLogger().addHandler(self._log_handler)
         self._live = Live(
-            self._render_dashboard(),
+            _DashboardRenderable(self),
             console=self._console,
             refresh_per_second=8,
-            transient=False,
+            screen=True,
         )
         self._live.start()
-        self._progress.start()
-        self._refresh_dashboard()
+        self._logger.info("CLI dashboard started")
         return self
 
     def __exit__(
@@ -97,69 +118,54 @@ class CLI(AbstractContextManager):
         exc: BaseException | None,
         tb: TracebackType | None,
     ) -> Literal[False]:
-        if self._live:
-            self._live.stop()
-        if self._progress:
-            self._progress.stop()
-        if self._log_handler:
-            logging.getLogger().removeHandler(self._log_handler)
-            self._log_handler.close()
+        live = getattr(self, "_live", None)
+        if live is not None:
+            live.stop()
+        self._teardown_logging()
         return False
 
-    def run(self, *args: Any, **kwargs: Any) -> Any:
-        self._logger.info("CLI runner starting")
-
-        forwarded = dict(kwargs)
-        forwarded.setdefault("progress_hook", self._progress_hook)
-        forwarded.setdefault("worker_status_hook", self._status_hook)
-
-        result = self._runner.run(*args, **forwarded)
-
-        self._logger.info("CLI runner finished")
-        return result
+    # RunObserver interface
 
-    def _progress_hook(self, done: int, total: int) -> None:
-        # Initialize task on first call
+    def on_progress(self, done: int, total: int) -> None:
         if self._progress_task_id is None:
-            description = "Processing data"
-            self._progress_task_id = self._progress.add_task(description, total=total)
-
-        if self._progress_task_id is not None:
-            self._progress.update(self._progress_task_id, completed=done, total=total)
-        self._refresh_dashboard()
-
-    def _status_hook(self, snapshot: dict[str, list[dict[str, Any]]]) -> None:
-        self._solver_statuses = snapshot.get("solvers", [])
-        self._checker_statuses = snapshot.get("checkers", [])
-        self._refresh_dashboard()
+            self._progress_task_id = self._progress.add_task(
+                "Processing data", total=total
+            )
+        self._progress.update(self._progress_task_id, completed=done, total=total)
+
+    def on_worker_status(self, snapshot: WorkerSnapshot) -> None:
+        self._solver_statuses = list(snapshot.solvers)
+        self._checker_statuses = list(snapshot.checkers)
+
+    # Internal
+
+    def _teardown_logging(self) -> None:
+        root = logging.getLogger()
+        log_handler = getattr(self, "_log_handler", None)
+        if log_handler is not None:
+            root.removeHandler(log_handler)
+            log_handler.close()
+        file_handler = getattr(self, "_file_handler", None)
+        if file_handler is not None:
+            root.removeHandler(file_handler)
+            file_handler.close()
+        teardown_session_log = getattr(self, "_teardown_session_log", None)
+        if teardown_session_log is not None:
+            teardown_session_log()
 
     def _setup_logging(self) -> None:
         self._log_file.parent.mkdir(parents=True, exist_ok=True)
 
-        logging.config.dictConfig(
-            {
-                "version": 1,
-                "disable_existing_loggers": False,
-                "formatters": {
-                    "default": {
-                        "format": "%(asctime)s %(name)s %(levelname)s %(message)s",
-                    }
-                },
-                "handlers": {
-                    "file": {
-                        "class": "logging.FileHandler",
-                        "level": self._log_level,
-                        "formatter": "default",
-                        "filename": str(self._log_file),
-                        "encoding": "utf-8",
-                    }
-                },
-                "root": {
-                    "level": self._log_level,
-                    "handlers": ["file"],
-                },
-            }
+        self._file_handler = logging.FileHandler(self._log_file, encoding="utf-8")
+        self._file_handler.setLevel(self._log_level)
+        self._file_handler.setFormatter(
+            logging.Formatter("%(asctime)s %(name)s %(levelname)s %(message)s")
         )
+        root = logging.getLogger()
+        root.setLevel(self._log_level)
+        root.addHandler(self._file_handler)
+
+        self._teardown_session_log = configure_session_log_file(self._session_log_file)
 
     @staticmethod
     def _resolve_log_level(level: int | str) -> int:
@@ -180,14 +186,6 @@ class CLI(AbstractContextManager):
 
     def _add_log_entry(self, entry: dict[str, str]) -> None:
         self._log_buffer.append(entry)
-        self._refresh_dashboard()
-
-    def _refresh_dashboard(self) -> None:
-        try:
-            self._live.update(self._render_dashboard(), refresh=True)
-        except Exception:
-            # Avoid crashing the run due to rendering issues.
-            return
 
     def _render_dashboard(self) -> Layout:
         layout = Layout(name="root")
@@ -199,11 +197,11 @@ class CLI(AbstractContextManager):
         return layout
 
     def _render_progress_panel(self) -> Panel:
-        if self._progress is None or self._progress_task_id is None:
-            body: Any = "Waiting for progress updates..."
-        else:
-            body = self._progress
-
+        body: Any = (
+            self._progress
+            if self._progress_task_id is not None
+            else "Waiting for progress updates..."
+        )
         return Panel(body, title="Run Progress", box=box.SIMPLE)
 
     def _render_workers_panel(self) -> Panel:
@@ -225,13 +223,13 @@ class CLI(AbstractContextManager):
         if not self._solver_statuses:
             solvers.add_row("-", "-", "-", "-", "-")
         else:
-            for st in self._solver_statuses:
+            for solver_st in self._solver_statuses:
                 solvers.add_row(
-                    str(st.get("id", "-")),
-                    str(st.get("state", "-")),
-                    str(st.get("data_id", "-")),
-                    f"{st.get('curr_run', '-')}/{st.get('total_runs', '-')}",
-                    str(st.get("session_length", "-")),
+                    str(solver_st.id),
+                    str(solver_st.state),
+                    str(solver_st.data_id) if solver_st.data_id is not None else "-",
+                    f"{solver_st.curr_run}/{solver_st.total_runs}",
+                    str(solver_st.session_length),
                 )
 
         checkers = Table(
@@ -251,12 +249,16 @@ class CLI(AbstractContextManager):
         if not self._checker_statuses:
             checkers.add_row("-", "-", "-", "-")
         else:
-            for st in self._checker_statuses:
+            for checker_st in self._checker_statuses:
                 checkers.add_row(
-                    str(st.get("id", "-")),
-                    str(st.get("state", "-")),
-                    str(st.get("data_id", "-")),
-                    str(st.get("solver_id", "-")),
+                    str(checker_st.id),
+                    str(checker_st.state),
+                    str(checker_st.data_id) if checker_st.data_id is not None else "-",
+                    (
+                        str(checker_st.solver_id)
+                        if checker_st.solver_id is not None
+                        else "-"
+                    ),
                 )
 
         grid = Table.grid(expand=True)
@@ -276,23 +278,20 @@ class CLI(AbstractContextManager):
         )
         table.add_column("Time", style="dim", width=8, no_wrap=True)
         table.add_column("Level", width=8, no_wrap=True)
-        table.add_column("Logger", style="dim", width=20, no_wrap=True)
+        table.add_column("Logger", style="dim", width=14, no_wrap=True)
         table.add_column("Message", overflow="fold")
 
         if not self._log_buffer:
             table.add_row("-", "-", "-", "Waiting for logs...")
         else:
-            max_rows = 6
-            if self._console is not None:
-                max_rows = max(6, min(30, self._console.size.height - 12))
+            max_rows = max(6, min(30, self._console.size.height - 12))
 
             for entry in reversed(list(self._log_buffer)[-max_rows:]):
                 table.add_row(
                     entry.get("time", ""),
                     entry.get("level", ""),
                     entry.get("logger", ""),
-                    entry.get("message", ""),
+                    Text(entry.get("message", "")),
                 )
-                table.add_section()
 
         return Panel(table, title="Latest Logs", box=box.SIMPLE, padding=(0, 0))

{ape_framework-0.1.2 → ape_framework-2.0.0.dev1}/ape/combine.py

@@ -8,10 +8,8 @@ from pathlib import Path
 
 
 def _merge_batches(dst: dict, src: dict) -> None:
-    dst["solutions"].extend(src.get("solutions", []))
-    dst["solution_runtimes"].extend(src.get("solution_runtimes", []))
-    dst["solution_total_times"].extend(src.get("solution_total_times", []))
-    dst["checker_outputs"].extend(src.get("checker_outputs", []))
+    dst["_solver_results"].extend(src.get("_solver_results", []))
+    dst["_check_results"].extend(src.get("_check_results", []))
 
 
 def main() -> None:
@@ -59,7 +57,10 @@ def main() -> None:
                         continue
                     try:
                         output = json.loads(line)
-                        data_id = output["input_data"]["_Data__id"]
+                        # _input_data is a jsonpickle-encoded DataModel; its full
+                        # serialization is the merge key (the schema's id field
+                        # name is not known at this layer).
+                        data_id = json.dumps(output["_input_data"], sort_keys=True)
                         if data_id in merged_data:
                             _merge_batches(merged_data[data_id], output)
                         else:

ape_framework-2.0.0.dev1/ape/data/__init__.py

@@ -0,0 +1,10 @@
+from .data import DataModel, id_field
+from .data_provider import CSVDataProvider, DataProvider, JSONLDataProvider
+
+__all__ = [
+    "DataModel",
+    "id_field",
+    "DataProvider",
+    "CSVDataProvider",
+    "JSONLDataProvider",
+]