ape-framework 0.1.0__tar.gz

ape_framework-0.1.0/PKG-INFO

@@ -0,0 +1,50 @@
+Metadata-Version: 2.4
+Name: ape-framework
+Version: 0.1.0
+Summary: Package for evaluating algebra problems using AI systems.
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Dynamic: license-file
+
+# Algebra Problems Evaluator (APE)
+
+## What is APE?
+**APE is a framework that simplifies building and running mathematical benchmarks for AI systems.**  
+  
+It was initially meant as a way to evaluate mathematical problems from the field of algebra on whether they are suited to be a target goal for LLM reasoning research (hence the name).
+However it might be more useful to think about it the other way around - the subjects of the evaluation are LLMs or more broadly - AI systems
+and they are being evaluated on their ability to solve specific algebra problems. Problems, which solutions are hard to generate, but relatively easy to automatically check for correctness.  
+  
+APE was created as a part of a Bachelor's project at the University of Warsaw.
+
+## User's Guide
+[**TODO**]
+
+## Development setup
+
+Install development dependencies:
+
+```bash
+pip install -r requirements-dev.txt
+```
+
+Install git hooks:
+
+```bash
+pre-commit install
+```
+
+Run all hooks manually:
+
+```bash
+pre-commit run --all-files
+```
+
+<!-- 
+    1. What the user has to implement to have a complete benchmark?
+    2. How to run a test using an APE benchmark?
+    3. Where to find the documentation?
+-->

ape_framework-0.1.0/README.md

@@ -0,0 +1,39 @@
+# Algebra Problems Evaluator (APE)
+
+## What is APE?
+**APE is a framework that simplifies building and running mathematical benchmarks for AI systems.**  
+  
+It was initially meant as a way to evaluate mathematical problems from the field of algebra on whether they are suited to be a target goal for LLM reasoning research (hence the name).
+However it might be more useful to think about it the other way around - the subjects of the evaluation are LLMs or more broadly - AI systems
+and they are being evaluated on their ability to solve specific algebra problems. Problems, which solutions are hard to generate, but relatively easy to automatically check for correctness.  
+  
+APE was created as a part of a Bachelor's project at the University of Warsaw.
+
+## User's Guide
+[**TODO**]
+
+## Development setup
+
+Install development dependencies:
+
+```bash
+pip install -r requirements-dev.txt
+```
+
+Install git hooks:
+
+```bash
+pre-commit install
+```
+
+Run all hooks manually:
+
+```bash
+pre-commit run --all-files
+```
+
+<!-- 
+    1. What the user has to implement to have a complete benchmark?
+    2. How to run a test using an APE benchmark?
+    3. Where to find the documentation?
+-->

ape_framework-0.1.0/ape/__init__.py

@@ -0,0 +1,5 @@
+import logging
+
+# Prevent logging if the user did not configure it
+logger = logging.getLogger(__name__)
+logger.addHandler(logging.NullHandler())

ape_framework-0.1.0/ape/adapter.py

@@ -0,0 +1,11 @@
+from abc import ABC, abstractmethod
+from typing import Generic, TypeVar
+
+RawT = TypeVar("RawT")
+SolutionT = TypeVar("SolutionT")
+
+
+class SolutionAdapter(ABC, Generic[RawT, SolutionT]):
+    @classmethod
+    @abstractmethod
+    def adapt(cls, raw: RawT) -> list[SolutionT]: ...

ape_framework-0.1.0/ape/checker.py

@@ -0,0 +1,39 @@
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from datetime import timedelta
+from enum import Enum
+from typing import Any, Generic, Optional, Type, TypeVar
+
+from .adapter import SolutionT
+from .data import DataT
+
+ResultT = TypeVar("ResultT", bound=Enum)
+
+
+@dataclass(frozen=True)
+class CheckerOutput(Generic[ResultT]):
+    result: ResultT
+    runtime: Optional[timedelta]
+    # Additional metadata can be added
+
+
+class Checker(ABC, Generic[DataT, SolutionT, ResultT]):
+    @abstractmethod
+    def check(
+        self, data: DataT, solution_batch: list[SolutionT]
+    ) -> list[CheckerOutput[ResultT]]: ...
+
+
+CheckerT = TypeVar("CheckerT", bound=Checker)
+
+
+class CheckerFactory(Generic[CheckerT]):
+    def __init__(self, checker_cls: Type[CheckerT], **kwargs: object) -> None:
+        self._checker_cls = checker_cls
+        self._kwargs = kwargs
+
+    def create(self) -> CheckerT:
+        return self._checker_cls(**self._kwargs)
+
+    def get_params(self) -> dict[str, Any]:
+        return self._kwargs

ape_framework-0.1.0/ape/cli.py

@@ -0,0 +1,298 @@
+from __future__ import annotations
+
+import logging
+import logging.config
+from collections import deque
+from contextlib import AbstractContextManager
+from datetime import datetime
+from pathlib import Path
+from types import TracebackType
+from typing import Any, Callable, Literal, Optional
+
+from rich import box
+from rich.console import Console
+from rich.layout import Layout
+from rich.live import Live
+from rich.panel import Panel
+from rich.progress import (
+    BarColumn,
+    Progress,
+    SpinnerColumn,
+    TaskID,
+    TaskProgressColumn,
+    TextColumn,
+    TimeElapsedColumn,
+)
+from rich.table import Table
+
+from .runner import Runner
+
+
+class LogBufferHandler(logging.Handler):
+    def __init__(self, add_entry: Callable[[dict[str, str]], None]) -> None:
+        super().__init__()
+        self._add_entry = add_entry
+
+    def emit(self, record: logging.LogRecord) -> None:
+        try:
+            entry = {
+                "time": datetime.fromtimestamp(record.created).strftime("%H:%M:%S"),
+                "level": record.levelname,
+                "logger": record.name,
+                "message": record.getMessage(),
+            }
+            self._add_entry(entry)
+        except Exception:
+            self.handleError(record)
+
+
+class CLI(AbstractContextManager):
+    def __init__(
+        self,
+        runner: Runner,
+        *,
+        log_file: Path | str = Path("output/run.log"),
+        log_level: int | str = logging.INFO,
+    ) -> None:
+        self._runner = runner
+        self._log_file = Path(log_file)
+        self._log_level = self._resolve_log_level(log_level)
+        self._progress_task_id: Optional[TaskID] = None
+        self._log_buffer: deque[dict[str, str]] = deque(maxlen=50)
+        self._solver_statuses: list[dict[str, Any]] = []
+        self._checker_statuses: list[dict[str, Any]] = []
+
+    def __enter__(self) -> CLI:
+        self._setup_logging()
+        self._logger = logging.getLogger(__name__)
+        self._console = Console()
+        self._progress = Progress(
+            SpinnerColumn(),
+            TextColumn("{task.description}", justify="left"),
+            BarColumn(bar_width=None),
+            TaskProgressColumn(),
+            TextColumn("{task.completed}/{task.total}"),
+            TimeElapsedColumn(),
+            console=self._console,
+            transient=False,
+            expand=True,
+        )
+        self._log_handler = LogBufferHandler(self._add_log_entry)
+        self._log_handler.setLevel(self._log_level)
+        logging.getLogger().addHandler(self._log_handler)
+        self._live = Live(
+            self._render_dashboard(),
+            console=self._console,
+            refresh_per_second=8,
+            transient=False,
+        )
+        self._live.start()
+        self._progress.start()
+        self._refresh_dashboard()
+        return self
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        tb: TracebackType | None,
+    ) -> Literal[False]:
+        if self._live:
+            self._live.stop()
+        if self._progress:
+            self._progress.stop()
+        if self._log_handler:
+            logging.getLogger().removeHandler(self._log_handler)
+            self._log_handler.close()
+        return False
+
+    def run(self, *args: Any, **kwargs: Any) -> Any:
+        self._logger.info("CLI runner starting")
+
+        forwarded = dict(kwargs)
+        forwarded.setdefault("progress_hook", self._progress_hook)
+        forwarded.setdefault("worker_status_hook", self._status_hook)
+
+        result = self._runner.run(*args, **forwarded)
+
+        self._logger.info("CLI runner finished")
+        return result
+
+    def _progress_hook(self, done: int, total: int) -> None:
+        # Initialize task on first call
+        if self._progress_task_id is None:
+            description = "Processing data"
+            self._progress_task_id = self._progress.add_task(description, total=total)
+
+        if self._progress_task_id is not None:
+            self._progress.update(self._progress_task_id, completed=done, total=total)
+        self._refresh_dashboard()
+
+    def _status_hook(self, snapshot: dict[str, list[dict[str, Any]]]) -> None:
+        self._solver_statuses = snapshot.get("solvers", [])
+        self._checker_statuses = snapshot.get("checkers", [])
+        self._refresh_dashboard()
+
+    def _setup_logging(self) -> None:
+        self._log_file.parent.mkdir(parents=True, exist_ok=True)
+
+        logging.config.dictConfig(
+            {
+                "version": 1,
+                "disable_existing_loggers": False,
+                "formatters": {
+                    "default": {
+                        "format": "%(asctime)s %(name)s %(levelname)s %(message)s",
+                    }
+                },
+                "handlers": {
+                    "file": {
+                        "class": "logging.FileHandler",
+                        "level": self._log_level,
+                        "formatter": "default",
+                        "filename": str(self._log_file),
+                        "encoding": "utf-8",
+                    }
+                },
+                "root": {
+                    "level": self._log_level,
+                    "handlers": ["file"],
+                },
+            }
+        )
+
+    @staticmethod
+    def _resolve_log_level(level: int | str) -> int:
+        if isinstance(level, int):
+            return level
+
+        normalized = level.strip().upper()
+        level_map = logging.getLevelNamesMapping()
+        if normalized in level_map:
+            return level_map[normalized]
+
+        if normalized.isdigit():
+            return int(normalized)
+
+        raise ValueError(
+            f"Invalid log level: {level!r}. Use a valid logging level name or integer."
+        )
+
+    def _add_log_entry(self, entry: dict[str, str]) -> None:
+        self._log_buffer.append(entry)
+        self._refresh_dashboard()
+
+    def _refresh_dashboard(self) -> None:
+        try:
+            self._live.update(self._render_dashboard(), refresh=True)
+        except Exception:
+            # Avoid crashing the run due to rendering issues.
+            return
+
+    def _render_dashboard(self) -> Layout:
+        layout = Layout(name="root")
+        layout.split_column(
+            Layout(self._render_progress_panel(), name="progress", size=4),
+            Layout(self._render_workers_panel(), name="workers", ratio=2),
+            Layout(self._render_logs_panel(), name="logs", ratio=3),
+        )
+        return layout
+
+    def _render_progress_panel(self) -> Panel:
+        if self._progress is None or self._progress_task_id is None:
+            body: Any = "Waiting for progress updates..."
+        else:
+            body = self._progress
+
+        return Panel(body, title="Run Progress", box=box.SIMPLE)
+
+    def _render_workers_panel(self) -> Panel:
+        solvers = Table(
+            show_header=True,
+            header_style="bold",
+            expand=True,
+            box=box.SIMPLE,
+            padding=(0, 0),
+            title="Solvers",
+            title_style="bold",
+        )
+        solvers.add_column("Solver", justify="center")
+        solvers.add_column("State", justify="center")
+        solvers.add_column("Data", justify="center")
+        solvers.add_column("Runs completed", justify="center")
+        solvers.add_column("Session length", justify="center")
+
+        if not self._solver_statuses:
+            solvers.add_row("-", "-", "-", "-", "-")
+        else:
+            for st in self._solver_statuses:
+                solvers.add_row(
+                    str(st.get("id", "-")),
+                    str(st.get("state", "-")),
+                    str(st.get("data_id", "-")),
+                    f"{st.get('curr_run', '-')}/{st.get('total_runs', '-')}",
+                    str(st.get("session_length", "-")),
+                )
+
+        checkers = Table(
+            show_header=True,
+            header_style="bold",
+            expand=True,
+            box=box.SIMPLE,
+            padding=(0, 0),
+            title="Checkers",
+            title_style="bold",
+        )
+        checkers.add_column("Checker", justify="center")
+        checkers.add_column("State", justify="center")
+        checkers.add_column("Data", justify="center")
+        checkers.add_column("From solver", justify="center")
+
+        if not self._checker_statuses:
+            checkers.add_row("-", "-", "-", "-")
+        else:
+            for st in self._checker_statuses:
+                checkers.add_row(
+                    str(st.get("id", "-")),
+                    str(st.get("state", "-")),
+                    str(st.get("data_id", "-")),
+                    str(st.get("solver_id", "-")),
+                )
+
+        grid = Table.grid(expand=True)
+        grid.add_column(ratio=1)
+        grid.add_column(ratio=1)
+        grid.add_row(solvers, checkers)
+
+        return Panel(grid, box=box.SIMPLE)
+
+    def _render_logs_panel(self) -> Panel:
+        table = Table(
+            show_header=True,
+            header_style="bold",
+            expand=True,
+            box=box.SIMPLE,
+            padding=(0, 0),
+        )
+        table.add_column("Time", style="dim", width=8, no_wrap=True)
+        table.add_column("Level", width=8, no_wrap=True)
+        table.add_column("Logger", style="dim", width=20, no_wrap=True)
+        table.add_column("Message", overflow="fold")
+
+        if not self._log_buffer:
+            table.add_row("-", "-", "-", "Waiting for logs...")
+        else:
+            max_rows = 6
+            if self._console is not None:
+                max_rows = max(6, min(30, self._console.size.height - 12))
+
+            for entry in reversed(list(self._log_buffer)[-max_rows:]):
+                table.add_row(
+                    entry.get("time", ""),
+                    entry.get("level", ""),
+                    entry.get("logger", ""),
+                    entry.get("message", ""),
+                )
+                table.add_section()
+
+        return Panel(table, title="Latest Logs", box=box.SIMPLE, padding=(0, 0))

ape_framework-0.1.0/ape/combine.py

@@ -0,0 +1,93 @@
+import argparse
+import json
+import sys
+from pathlib import Path
+
+# We use raw jsons to avoid issues with unpickling types etc.
+# This file needs to be updated for each API breaking change, but it works nicely
+
+
+def _merge_batches(dst: dict, src: dict) -> None:
+    dst["solutions"].extend(src.get("solutions", []))
+    dst["solution_runtimes"].extend(src.get("solution_runtimes", []))
+    dst["solution_total_times"].extend(src.get("solution_total_times", []))
+    dst["checker_outputs"].extend(src.get("checker_outputs", []))
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Combine APE run results.")
+    parser.add_argument("folder", help="Folder containing .jsonl run files")
+    args = parser.parse_args()
+
+    folder = Path(args.folder)
+    if not folder.exists() or not folder.is_dir():
+        print(f"Error: {folder} does not exist or is not a directory.", file=sys.stderr)
+        sys.exit(1)
+
+    output_path = folder / "combined.jsonl"
+
+    if output_path.exists():
+        print(
+            f"Error: Output file {output_path} already exists. Aborting to prevent overwriting.",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+
+    input_files = []
+    for f in folder.glob("*.jsonl"):
+        input_files.append(f)
+
+    if not input_files:
+        print("No .jsonl files found to combine.")
+        return
+
+    # Sort files for deterministic order
+    input_files.sort()
+
+    print(f"Found {len(input_files)} files to combine into {output_path}")
+
+    merged_data: dict[str, dict] = {}
+
+    def process_file(fpath: Path) -> None:
+        print(f"Reading {fpath.name}...")
+        count = 0
+        try:
+            with open(fpath, "r") as infile:
+                for line in infile:
+                    line = line.strip()
+                    if not line:
+                        continue
+                    try:
+                        output = json.loads(line)
+                        data_id = output["input_data"]["_Data__id"]
+                        if data_id in merged_data:
+                            _merge_batches(merged_data[data_id], output)
+                        else:
+                            merged_data[data_id] = output
+                        count += 1
+                    except Exception as e:
+                        print(
+                            f"  Error decoding line in {fpath.name}: {e}",
+                            file=sys.stderr,
+                        )
+            print(f"  Processed {count} entries.")
+        except Exception as e:
+            print(f"  Error reading {fpath.name}: {e}", file=sys.stderr)
+
+    for fpath in input_files:
+        process_file(fpath)
+
+    print(f"Writing merged results ({len(merged_data)} items) to {output_path}...")
+    try:
+        with open(output_path, "w") as outfile:
+            for output in merged_data.values():
+                outfile.write(json.dumps(output) + "\n")
+    except Exception as e:
+        print(f"Error writing to {output_path}: {e}", file=sys.stderr)
+        sys.exit(1)
+
+    print("Done.")
+
+
+if __name__ == "__main__":
+    main()

ape_framework-0.1.0/ape/report.py

@@ -0,0 +1,23 @@
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Any, Generic
+
+from .adapter import SolutionT
+from .checker import ResultT
+from .data import DataT
+from .runner import Output
+
+
+class Report(ABC, Generic[DataT, SolutionT, ResultT]):
+
+    # JSON-like dictionary with data in a Typst templates compatible format.
+    _summarized_data: dict[str, Any]
+
+    @abstractmethod
+    def __init__(self, output: Output[DataT, SolutionT, ResultT]) -> None: ...
+
+    @abstractmethod
+    def save(self, path: Path) -> None: ...
+
+    @abstractmethod
+    def dump_data(self, path: Path) -> None: ...

ape_framework-0.1.0/ape/save_handler.py

@@ -0,0 +1,122 @@
+import logging
+import os
+import sys
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Generic, Optional, cast
+
+import jsonpickle
+
+from .adapter import SolutionT
+from .checker import ResultT
+from .data import Data, SchemaT
+from .runner.output import OutputBatch
+from .runner.run_params import RunParams
+
+logger = logging.getLogger(__name__)
+
+
+class SaveHandler(Generic[SchemaT, SolutionT, ResultT]):
+    """Responsible for handling the result file; so loading the save if it is present,
+    and appending new output to it."""
+
+    def __init__(
+        self,
+        run_params: RunParams,
+        solver_params: dict[str, Any],
+        checker_params: dict[str, Any],
+    ):
+        """Create a new save handler. If the output file exists, look for a save
+        in it for cross-run persistence. If a save is present, the last saved data id is
+        found and can be retrieved by calling `last_from_save`. The save format is JSONL for the output
+        batches and JSON for run metadata.
+        """
+        self._save_file: Path = run_params.save_file
+        self._metadata_file: Path = run_params.metadata_file
+        self._last: Optional[str] = None
+
+        self._save_file.parent.mkdir(parents=True, exist_ok=True)
+        self._metadata_file.parent.mkdir(parents=True, exist_ok=True)
+
+        if run_params.load_save:
+            try:
+                metadata = cast(
+                    dict[str, Any], jsonpickle.decode(self._metadata_file.read_text())
+                )
+                _ = self._save_file.read_text()  # Run to ensure the file exists
+
+                last_id = metadata.get("last_id")
+                self._last = str(last_id) if last_id is not None else None
+
+                logger.info(f"Save found and loaded. Last label: {self._last}.")
+                return
+            except FileNotFoundError:
+                logger.info("No save found.")
+            except Exception as e:
+                logger.warning(
+                    f"Tried to read save from file {self._save_file}, but the save is corrupted. "
+                    f"Error encountered: {e}."
+                )
+
+        logger.info("Creating new save files.")
+        try:
+            self._save_file.touch(exist_ok=False)
+        except FileExistsError:
+            logger.critical(
+                "Found save files when creating new files. Stopping to prevent data overwriting."
+            )
+            raise RuntimeError
+
+        metadata = {
+            "run_params": run_params,
+            "solver_params": solver_params,
+            "checker_params": checker_params,
+            "start_date": datetime.now(),
+        }
+        metadata_payload = cast(
+            str,
+            jsonpickle.encode(metadata, unpicklable=True, make_refs=False),
+        )
+        self._metadata_file.write_text(metadata_payload)
+
+    def last_from_save(self) -> str | None:
+        """See which Data id was the last calculated in the previous run.
+
+        Returns:
+            id (str | None): the last id (or None if no save loaded).
+        """
+        return self._last
+
+    def save(
+        self,
+        output_batch: OutputBatch[Data[SchemaT], SolutionT, ResultT],
+    ) -> None:
+        """Add the new results to the save file.
+
+        Params:
+            data (Data[SchemaT]): the data item the given batch is for
+            solution_result_pairs (list[tuple[SolutionT, ResultT]]): a list of
+                corresponding pairs (solution, result)
+        """
+        metadata = cast(
+            dict[str, Any], jsonpickle.decode(self._metadata_file.read_text())
+        )
+        metadata["last_id"] = output_batch.input_data.id
+        metadata["last_time"] = datetime.now()
+        total = metadata.get("total", 0)
+        metadata["total"] = int(total) + 1
+
+        metadata_payload = cast(
+            str,
+            jsonpickle.encode(metadata, unpicklable=True, make_refs=False),
+        )
+        self._metadata_file.write_text(metadata_payload)
+
+        with open(self._save_file, "a") as f:
+            output_payload = cast(
+                str,
+                jsonpickle.encode(output_batch, unpicklable=True, make_refs=False),
+            )
+            f.write(output_payload + "\n")
+            f.flush()
+            os.fsync(f.fileno())

ape_framework-0.1.0/ape/solver.py

@@ -0,0 +1,46 @@
+from abc import ABC, abstractmethod
+from typing import Any, Generic, Optional, Type, TypeVar
+
+from .adapter import RawT
+from .checker import CheckerOutput, ResultT
+from .data import DataT
+
+
+class Solver(ABC, Generic[DataT, RawT, ResultT]):
+    @abstractmethod
+    def begin_session(self, data: DataT) -> RawT:
+        """Starts the solving session and returns a first raw solution.
+        Should only be called right after construction, or after advance_session returned None.
+        """
+        ...
+
+    @abstractmethod
+    def advance_session(self, outputs: list[CheckerOutput[ResultT]]) -> Optional[RawT]:
+        """Receives the results of the previous solution and tries to make another one,
+        or decides to stop here and return None.
+        Should not be called again after returning None, or not until begin_session was called again.
+        """
+        ...
+
+    @abstractmethod
+    def retry_last(self) -> RawT:
+        """Tries to generate the solution again, in an event that the previous raw solution
+        could not be parsed.
+        Should only be called after a call to begin_session or advance_session, if it returned something.
+        """
+        ...
+
+
+SolverT = TypeVar("SolverT", bound=Solver)
+
+
+class SolverFactory(Generic[SolverT]):
+    def __init__(self, solver_cls: Type[SolverT], **kwargs: object) -> None:
+        self._solver_cls = solver_cls
+        self._kwargs = kwargs
+
+    def create(self) -> SolverT:
+        return self._solver_cls(**self._kwargs)
+
+    def get_params(self) -> dict[str, Any]:
+        return self._kwargs

ape_framework-0.1.0/ape/visualizer.py

@@ -0,0 +1,67 @@
+import sys
+from typing import Any, cast
+
+import jsonpickle
+
+from .runner.output import OutputBatch
+from .runner.run_params import RunParams
+
+
+# A very simple visualizer for now
+class Visualizer:
+    def visualize(self, run_params: RunParams) -> None:
+        save_f = run_params.save_file
+        if not save_f.exists() or not save_f.is_file():
+            print(f"Error: {save_f} does not exist or is not a file.", file=sys.stderr)
+            return
+
+        metadata_f = run_params.metadata_file
+        if not metadata_f.exists() or not metadata_f.is_file():
+            print(
+                f"Error: {metadata_f} does not exist or is not a file.", file=sys.stderr
+            )
+            return
+
+        print(f"Reading metadata from {metadata_f}...\n")
+        try:
+            metadata = cast(dict[str, Any], jsonpickle.decode(metadata_f.read_text()))
+            for k, v in metadata.items():
+                print(f"  {k}: {v}")
+        except Exception as e:
+            print(f"Error decoding metadata: {e}", file=sys.stderr)
+
+        print(f"\n\nVisualizing results from {save_f}...\n")
+        try:
+            with open(save_f, "r") as f:
+                for i, line in enumerate(f):
+                    line = line.strip()
+                    if not line:
+                        continue
+                    try:
+                        output = cast(
+                            OutputBatch[Any, Any, Any], jsonpickle.decode(line)
+                        )
+                        print(f"--- Entry {i + 1} ---")
+                        print(f"Data:\n{output.input_data}")
+                        print("\nResults:")
+                        if len(output.solutions) > 0:
+                            for j, (
+                                sol,
+                                gen_time,
+                                total_gen_time,
+                                checker_output,
+                            ) in enumerate(output):
+                                print(f"  Result {j + 1}:")
+                                print(f"    Solution: {sol}")
+                                print(f"    Status:   {checker_output.result}")
+                                print(f"    Checking time: {checker_output.runtime}")
+                                print(f"    Generation time: {gen_time}")
+                                print(f"    Total generation time: {total_gen_time}")
+                        else:
+                            print("  No results found.")
+                        print("\n" + "=" * 40 + "\n")
+
+                    except Exception as e:
+                        print(f"Error decoding line {i + 1}: {e}", file=sys.stderr)
+        except Exception as e:
+            print(f"Error reading file {save_f}: {e}", file=sys.stderr)

ape_framework-0.1.0/ape_framework.egg-info/PKG-INFO

@@ -0,0 +1,50 @@
+Metadata-Version: 2.4
+Name: ape-framework
+Version: 0.1.0
+Summary: Package for evaluating algebra problems using AI systems.
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Dynamic: license-file
+
+# Algebra Problems Evaluator (APE)
+
+## What is APE?
+**APE is a framework that simplifies building and running mathematical benchmarks for AI systems.**  
+  
+It was initially meant as a way to evaluate mathematical problems from the field of algebra on whether they are suited to be a target goal for LLM reasoning research (hence the name).
+However it might be more useful to think about it the other way around - the subjects of the evaluation are LLMs or more broadly - AI systems
+and they are being evaluated on their ability to solve specific algebra problems. Problems, which solutions are hard to generate, but relatively easy to automatically check for correctness.  
+  
+APE was created as a part of a Bachelor's project at the University of Warsaw.
+
+## User's Guide
+[**TODO**]
+
+## Development setup
+
+Install development dependencies:
+
+```bash
+pip install -r requirements-dev.txt
+```
+
+Install git hooks:
+
+```bash
+pre-commit install
+```
+
+Run all hooks manually:
+
+```bash
+pre-commit run --all-files
+```
+
+<!-- 
+    1. What the user has to implement to have a complete benchmark?
+    2. How to run a test using an APE benchmark?
+    3. Where to find the documentation?
+-->

ape_framework-0.1.0/ape_framework.egg-info/SOURCES.txt

@@ -0,0 +1,16 @@
+LICENSE
+README.md
+pyproject.toml
+ape/__init__.py
+ape/adapter.py
+ape/checker.py
+ape/cli.py
+ape/combine.py
+ape/report.py
+ape/save_handler.py
+ape/solver.py
+ape/visualizer.py
+ape_framework.egg-info/PKG-INFO
+ape_framework.egg-info/SOURCES.txt
+ape_framework.egg-info/dependency_links.txt
+ape_framework.egg-info/top_level.txt

ape_framework-0.1.0/ape_framework.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

ape_framework-0.1.0/ape_framework.egg-info/top_level.txt

@@ -0,0 +1 @@
+ape

ape_framework-0.1.0/pyproject.toml

@@ -0,0 +1,37 @@
+[tool.black]
+line-length = 88
+target-version = ["py311"]
+skip-string-normalization = false
+
+[tool.isort]
+profile = "black"
+line_length = 88
+
+[tool.mypy]
+python_version = "3.12"
+packages = ["ape"]
+disallow_untyped_defs = true
+disallow_incomplete_defs = true
+no_implicit_optional = true
+warn_return_any = true
+warn_unused_ignores = true
+strict_equality = true
+ignore_missing_imports = true
+
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "ape-framework"
+version = "0.1.0"
+description = "Package for evaluating algebra problems using AI systems."
+readme = "README.md"
+requires-python = ">=3.12"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Operating System :: OS Independent"
+]
+
+[tool.setuptools]
+packages = ["ape"]

ape_framework-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+