verily 0.1.0__tar.gz

verily-0.1.0/.gitignore

@@ -0,0 +1,197 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# Abstra
+# Abstra is an AI-powered process automation framework.
+# Ignore directories containing user credentials, local state, and settings.
+# Learn more at https://abstra.io/docs
+.abstra/
+
+# Visual Studio Code
+#  Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore 
+#  that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
+#  and can be added to the global gitignore or merged into this file. However, if you prefer, 
+#  you could uncomment the following to ignore the enitre vscode folder
+# .vscode/
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# Cursor
+#  Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to
+#  exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
+#  refer to https://docs.cursor.com/context/ignore-files
+.cursorignore
+.cursorindexingignore
+
+# macOS files
+.DS_Store

verily-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 wowthx
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

verily-0.1.0/PKG-INFO

@@ -0,0 +1,17 @@
+Metadata-Version: 2.4
+Name: verily
+Version: 0.1.0
+Summary: Simple stability testing test for stochastic systems
+License-File: LICENSE
+Requires-Python: >=3.11
+Requires-Dist: aiostream>=0.6.4
+Requires-Dist: fsspec>=2025.5.1
+Requires-Dist: numpy>=2.3.1
+Requires-Dist: pydantic>=2.11.7
+Requires-Dist: rich>=14.0.0
+Requires-Dist: scipy>=1.16.0
+Requires-Dist: typer>=0.16.0
+Description-Content-Type: text/markdown
+
+# Verily
+Simple stability testing test for stochastic systems

verily-0.1.0/README.md

@@ -0,0 +1,2 @@
+# Verily
+Simple stability testing test for stochastic systems

verily-0.1.0/pyproject.toml

@@ -0,0 +1,69 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "verily"
+version = "0.1.0"
+description = "Simple stability testing test for stochastic systems"
+readme = "README.md"
+requires-python = ">=3.11"
+dependencies = [
+    "aiostream>=0.6.4",
+    "fsspec>=2025.5.1",
+    "numpy>=2.3.1",
+    "pydantic>=2.11.7",
+    "rich>=14.0.0",
+    "scipy>=1.16.0",
+    "typer>=0.16.0",
+]
+
+[project.scripts]
+zen = "verily.main:app"
+
+[dependency-groups]
+dev = [
+    "build>=1.2.2.post1",
+    "pytest>=8.4.0",
+    "pytest-asyncio>=1.0.0",
+    "ruff>=0.11.13",
+]
+
+[tool.poe.tasks]
+lint = "ruff check ."
+format = "ruff format ."
+test = "uv run pytest"
+build = "uv run pyproject-build"
+
+[tool.pytest.ini_options]
+pythonpath = ["src"]
+asyncio_mode = "auto"
+
+[tool.ruff.lint]
+extend-select = [
+    "F",        # Pyflakes rules
+    "W",        # PyCodeStyle warnings
+    "E",        # PyCodeStyle errors
+    "I",        # Sort imports properly
+    "UP",       # Warn if certain things can changed due to newer Python versions
+    "C4",       # Catch incorrect use of comprehensions, dict, list, etc
+    "FA",       # Enforce from __future__ import annotations
+    "ISC",      # Good use of string concatenation
+    "ICN",      # Use common import conventions
+    "RET",      # Good return practices
+    "SIM",      # Common simplification rules
+    "TID",      # Some good import practices
+    "TC",       # Enforce importing certain types in a TYPE_CHECKING block
+    "PTH",      # Use pathlib instead of os.path
+    "TD",       # Be diligent with TODO comments
+    "NPY",      # Some numpy-specific things
+]
+
+[tool.ruff.lint.flake8-tidy-imports.banned-api."pytest.mark.asyncio"]
+msg = "The @pytest.mark.asyncio decorator is not needed as `asyncio_mode` is set to `auto` for the project."
+
+[tool.hatch.build.targets.sdist]
+only-include = ["src/verily"]
+
+[tool.hatch.build.targets.wheel]
+only-include = ["src/verily"]

verily-0.1.0/src/verily/__init__.py

@@ -0,0 +1,3 @@
+from .models import Benchmark, BenchmarkCase, RunConfigOverride
+
+__all__ = ["Benchmark", "BenchmarkCase", "RunConfigOverride"]

verily-0.1.0/src/verily/display.py

@@ -0,0 +1,112 @@
+from rich.spinner import Spinner
+from rich.table import Table
+from rich.text import Text
+
+from .models import BenchmarkRunResult
+
+
+def generate_benchmark_table(
+    results: list[BenchmarkRunResult], confidence_level: float
+) -> Table:
+    table = Table(show_header=True, header_style="cyan", expand=True)
+    table.add_column("Benchmark", style="dim", width=30)
+    table.add_column(f"Mean @ {confidence_level:.0%} CI", justify="right")
+    table.add_column("Std Dev", justify="right")
+    table.add_column("Stability", justify="right")
+    table.add_column("Runs", justify="right")
+    table.add_column("Avg Dur~ (s)", justify="right")
+
+    if not results:
+        table.add_row("Waiting for benchmarks to start...", "", "", "", "", "")
+        return table
+
+    for result in results:
+        benchmark_name = result.benchmark.name
+        all_runs = [run for case in result.case_results for run in case.runs]
+
+        pending_runs = [r for r in all_runs if r.result.type == "pending"]
+        successful_runs = [r for r in all_runs if r.result.type == "success"]
+        failed_runs = [r for r in all_runs if r.result.type == "failure"]
+
+        latest_stats = result.stats
+
+        all_cases_done = all(cr.done for cr in result.case_results)
+
+        def with_spinner(text: str | None) -> str | Spinner | Text:
+            if pending_runs and not all_cases_done:
+                if text:
+                    return Spinner("dots", text=Text(text, style="white"), style="cyan")
+                return Spinner("dots", style="cyan")
+            return text or "[yellow]N/A[/yellow]"
+
+        mean_display_text = None
+        if latest_stats:
+            mean_val_text = f"{latest_stats.mean * 100:.2f}%"
+            if latest_stats.precision:
+                margin_of_error = (
+                    latest_stats.precision.high - latest_stats.precision.low
+                ) / 2
+                margin_of_error_text = f"±{margin_of_error * 100:.2f}%"
+                mean_display_text = f"{mean_val_text} ({margin_of_error_text})"
+            else:
+                mean_display_text = mean_val_text
+
+        std_dev_text = f"{latest_stats.std_dev:.3f}" if latest_stats else None
+        stability_text = f"{latest_stats.stability * 100:.2f}%" if latest_stats else None
+
+        mean_score_display = with_spinner(mean_display_text)
+        std_dev_display = with_spinner(std_dev_text)
+        stability_display = with_spinner(stability_text)
+
+        total_runs_count = len(all_runs)
+        runs_display_text = f"{total_runs_count}"
+        if failed_runs:
+            runs_display_text += f" [red]({len(failed_runs)} failed)[/red]"
+
+        avg_run_duration_display: str | Spinner | Text
+        avg_runtime_text = None
+        completed_runs = successful_runs + failed_runs
+        if completed_runs:
+            total_runtime = sum(r.result.runtime for r in completed_runs)
+            avg_runtime = total_runtime / len(completed_runs)
+            avg_runtime_text = f"{avg_runtime:.2f}"
+
+        avg_run_duration_display = with_spinner(avg_runtime_text)
+
+        table.add_row(
+            benchmark_name,
+            mean_score_display,
+            std_dev_display,
+            stability_display,
+            runs_display_text,
+            avg_run_duration_display,
+        )
+
+    return table
+
+
+def generate_failures_table(
+    results: list[BenchmarkRunResult]
+) -> Table:
+    table = Table(show_header=True, header_style="bold red")
+    table.title = "Failures"
+    table.add_column("Benchmark", style="dim", width=30)
+    table.add_column("Case", justify="right")
+    table.add_column("Error")
+    table.add_column("Stdout")
+    table.add_column("Stderr")
+
+    for result in results:
+        for case_result in result.case_results:
+            for run in case_result.runs:
+                if run.result.type == "failure":
+                    failure = run.result
+                    table.add_row(
+                        result.benchmark.name,
+                        str(case_result.case.inputs),
+                        failure.error_message,
+                        failure.stdout.decode("utf-8", errors="ignore"),
+                        failure.stderr.decode("utf-8", errors="ignore"),
+                    )
+
+    return table

verily-0.1.0/src/verily/io.py

@@ -0,0 +1,152 @@
+import json
+from collections.abc import Callable, Iterator
+from contextlib import contextmanager
+
+import fsspec
+
+from .models import (
+    BenchmarkCase,
+    BenchmarkRunResult,
+    BenchmarkStatistics,
+    CaseResult,
+    FailureResult,
+    NamedBenchmark,
+    Run,
+    RunConfig,
+    RunResult,
+    SuccessResult,
+)
+from .stats import aggregate_benchmark_statistics, calculate_run_stats
+from .utils import create_sequential_run_directory
+
+
+def _serialize_run(run: Run) -> dict:
+    result_dump = None
+    if hasattr(run.result, "model_dump"):
+        result_dump = run.result.model_dump()
+        if "stdout" in result_dump and isinstance(result_dump["stdout"], bytes):
+            result_dump["stdout"] = result_dump["stdout"].decode("utf-8", "ignore")
+        if "stderr" in result_dump and isinstance(result_dump["stderr"], bytes):
+            result_dump["stderr"] = result_dump["stderr"].decode("utf-8", "ignore")
+
+    return {
+        "result": result_dump,
+        "stats": run.stats.model_dump() if run.stats else None,
+    }
+
+
+@contextmanager
+def create_results_writer(
+    results_dir: str,
+    config: RunConfig,
+) -> Iterator[Callable[[BenchmarkRunResult], None]]:
+    fs, path = fsspec.url_to_fs(results_dir)
+    run_dir = create_sequential_run_directory(fs=fs, base_path=path)
+    written_runs = set()
+
+    with fs.open(f"{run_dir}/config.json", "w") as f:
+        json.dump(config.model_dump(), f, indent=4)
+
+    with fs.open(f"{run_dir}/results.jsonl", "w") as f:
+
+        def writer(result: BenchmarkRunResult) -> None:
+            benchmark_name = result.benchmark.name
+            for case_idx, case_result in enumerate(result.case_results):
+                for run_idx, run in enumerate(case_result.runs):
+                    run_id = (benchmark_name, case_idx, run_idx)
+                    if run.result.type != "pending" and run_id not in written_runs:
+                        run_data = _serialize_run(run)
+                        output_record = {
+                            "benchmark_name": benchmark_name,
+                            "case_index": case_idx,
+                            "case_inputs": case_result.case.inputs,
+                            "run_index": run_idx,
+                            **run_data,
+                        }
+                        f.write(json.dumps(output_record) + "\n")
+                        written_runs.add(run_id)
+
+        yield writer
+
+
+def load_results(run_path: str) -> tuple[list[BenchmarkRunResult], RunConfig]:
+    fs, path = fsspec.url_to_fs(run_path)
+
+    with fs.open(f"{path}/config.json", "r") as f:
+        config_dict = json.load(f)
+    config = RunConfig(**config_dict)
+
+    results_by_benchmark_case: dict[
+        tuple[str, int], list[Run]
+    ] = {}
+
+    with fs.open(f"{path}/results.jsonl", "r") as f:
+        for line in f:
+            record = json.loads(line)
+            benchmark_name = record["benchmark_name"]
+            case_index = record["case_index"]
+            case_inputs = record["case_inputs"]
+            run_index = record["run_index"]
+            run_data = record["result"]
+            stats_data = record["stats"]
+
+            run_result: RunResult
+            if run_data["type"] == "success":
+                run_result = SuccessResult(
+                    comparison=run_data["comparison"],
+                    stdout=run_data["stdout"].encode("utf-8"),
+                    stderr=run_data["stderr"].encode("utf-8"),
+                    runtime=run_data["runtime"],
+                )
+            elif run_data["type"] == "failure":
+                run_result = FailureResult(
+                    error_message=run_data["error_message"],
+                    stdout=run_data["stdout"].encode("utf-8"),
+                    stderr=run_data["stderr"].encode("utf-8"),
+                    runtime=run_data["runtime"],
+                )
+            else:
+                # This should not happen with current serialization logic
+                continue
+
+            run_stats = (
+                BenchmarkStatistics(**stats_data) if stats_data else None
+            )
+            run = Run(result=run_result, stats=run_stats)
+
+            if (benchmark_name, case_index) not in results_by_benchmark_case:
+                results_by_benchmark_case[(benchmark_name, case_index)] = []
+
+            # Ensure the list is long enough to insert at run_index
+            current_runs = results_by_benchmark_case[(benchmark_name, case_index)]
+            while len(current_runs) <= run_index:
+                current_runs.append(None) # type: ignore
+            current_runs[run_index] = run
+
+
+    benchmark_results: dict[str, BenchmarkRunResult] = {}
+
+    for (benchmark_name, case_index), runs in results_by_benchmark_case.items():
+        # NOTE: We don't have the original Benchmark object, so we create a dummy one
+        # This is acceptable as it's only used for display purposes and not execution
+        dummy_benchmark_case = BenchmarkCase(inputs=[], expectation=None)
+
+        # Filter out None runs and calculate stats for the case
+        successful_runs_in_case = [r.result for r in runs if r and r.result.type == "success"]
+        case_stats = calculate_run_stats(successful_runs_in_case, config.confidence_level)
+
+        case_result = CaseResult(case=dummy_benchmark_case, runs=runs, stats=case_stats)
+
+        if benchmark_name not in benchmark_results:
+            dummy_named_benchmark = NamedBenchmark(name=benchmark_name, benchmark=None) # type: ignore
+            benchmark_results[benchmark_name] = BenchmarkRunResult(
+                benchmark=dummy_named_benchmark, case_results=[], stats=None
+            )
+        benchmark_results[benchmark_name].case_results.append(case_result)
+
+    # Calculate overall benchmark statistics
+    for benchmark_run_result in benchmark_results.values():
+        all_case_stats = [cr.stats for cr in benchmark_run_result.case_results if cr.stats]
+        benchmark_run_result.stats = aggregate_benchmark_statistics(all_case_stats)
+
+    return list(benchmark_results.values()), config