apte 0.3.0__tar.gz

apte-0.3.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Renaud Cepre
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

apte-0.3.0/PKG-INFO

@@ -0,0 +1,211 @@
+Metadata-Version: 2.4
+Name: apte
+Version: 0.3.0
+Summary: Async-first Python testing framework
+Author-email: Renaud Cepre <renaudcepre@protonmail.com>
+License: MIT
+Project-URL: Homepage, https://github.com/renaudcepre/apte
+Project-URL: Documentation, https://renaudcepre.github.io/apte/
+Project-URL: Repository, https://github.com/renaudcepre/apte
+Project-URL: Changelog, https://github.com/renaudcepre/apte/blob/main/CHANGELOG.md
+Project-URL: Issues, https://github.com/renaudcepre/apte/issues
+Keywords: testing,async,dependency-injection,test-framework,asyncio,fixtures
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Testing
+Classifier: Framework :: AsyncIO
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: typing-extensions>=4.15.0
+Provides-Extra: rich
+Requires-Dist: rich>=15.0.0; extra == "rich"
+Provides-Extra: web
+Requires-Dist: websockets>=16.0; extra == "web"
+Dynamic: license-file
+
+![Apte](assets/logo-with-text-animate.svg)
+
+[![CI](https://github.com/renaudcepre/apte/actions/workflows/ci.yml/badge.svg)](https://github.com/renaudcepre/apte/actions/workflows/ci.yml)
+[![codecov](https://codecov.io/gh/renaudcepre/apte/graph/badge.svg?token=V0MLGEE5UZ)](https://codecov.io/gh/renaudcepre/apte)
+[![docs](https://img.shields.io/badge/docs-GitHub%20Pages-blue)](https://renaudcepre.github.io/apte/)
+
+**Write your tests and your LLM evals in one async-first framework.**
+
+An eval is just a test that returns a value - scored, not asserted. Your evals get
+real fixtures, dependency injection and parallelism, and live right next to the
+tests they ship with. Python 3.10+, installs lean (Rich UI optional).
+
+---
+
+## Why Apte?
+
+### Explicit Injection (IDE-Ready)
+
+**Ctrl+Click works.** Your IDE knows every type. No guessing where fixtures come from.
+
+```python
+def test_user(db: Annotated[Database, Use(database)]): ...
+```
+
+### Native Async & Parallelism
+
+Tests run as coroutines on a single event loop. No plugin needed.
+
+```bash
+apte run tests:session -n 10
+```
+
+### Smart Tagging (Tag Propagation)
+
+Tag a fixture once, every test using it inherits the tag automatically.
+
+```python
+@fixture(tags=["database"])
+def db(): ...
+
+session.bind(db)
+
+@session.test()
+def test_users(repo: Annotated[Repo, Use(user_repo)]): ...  # Also tagged "database"
+```
+
+```bash
+apte run tests:session --no-tag database  # Skips ALL tests touching DB
+```
+
+### Infra vs Code Errors (Error ≠ Fail)
+
+```
+✗ test_create_user: AssertionError       # Your bug - TEST FAILED
+⚠ test_with_db: [FIXTURE] ConnectionError  # Infra issue - SETUP ERROR
+```
+
+### Typed Parameterization
+
+```python
+CODES = ForEach([200, 201])
+
+@session.test()
+def test_status(code: Annotated[int, From(CODES)]): ...
+```
+
+### Native LLM Evals
+
+Score model outputs alongside your tests - same fixtures, same parallelism, same `apte` CLI. Cases get pass/fail + numeric metrics, persisted to JSONL for run-over-run comparison.
+
+```python
+from typing import Annotated
+from apte import ForEach, From, ApteSession
+from apte.evals import EvalCase, EvalSuite
+from apte.evals.evaluators import contains_keywords
+
+session = ApteSession()
+chatbot_suite = EvalSuite("chatbot")
+session.add_suite(chatbot_suite)
+
+cases = ForEach([
+    EvalCase(name="capital_fr", inputs="Capital of France?", expected="Paris"),
+])
+
+@chatbot_suite.eval(evaluators=[contains_keywords(keywords=["paris"])])
+async def chatbot(case: Annotated[EvalCase, From(cases)]) -> str:
+    return await my_agent(case.inputs)  # your LLM call
+```
+
+```bash
+apte eval evals.session:session   # runs are recorded to .apte/history.jsonl
+```
+
+See [Evals docs](https://renaudcepre.github.io/apte/evals/) for evaluators, judges, and scoring.
+
+---
+
+## Quick Start
+
+```python
+from apte import ApteSession
+
+session = ApteSession()
+
+
+def inc(x):
+    return x + 1
+
+
+@session.test()
+def test_answer():
+    assert inc(3) == 4
+```
+
+```bash
+apte run test_sample:session
+```
+
+## Installation
+
+Apte is not yet on PyPI. Install directly from GitHub:
+
+```bash
+# With uv (recommended)
+uv add git+https://github.com/renaudcepre/apte.git
+
+# With pip
+pip install git+https://github.com/renaudcepre/apte.git
+```
+
+## CLI
+
+```bash
+apte run module:session                    # Run tests
+apte run module:session -n 4               # Parallel (4 workers)
+apte run module:session --lf               # Re-run failed tests only
+apte run module:session --collect-only     # List tests without running
+apte run module:session --cache-clear      # Clear cache before run
+apte run module:session --app-dir src      # Look for module in src/
+apte run module:session --ctrf-output r.json  # CTRF report for CI/CD
+
+apte eval module:session                   # Run LLM evals
+apte eval module:session --tag safety      # Filter by case tag
+apte eval module:session --last-failed     # Re-run failed cases only
+```
+
+## Features
+
+- **Explicit DI** - No guessing which fixture you're using
+- **Async native** - No plugin needed, just `async def`
+- **Parallel execution** - Built-in with `-n 4`
+- **Scoped fixtures** - `SESSION`, `SUITE`, `TEST`
+- **Mix sync/async** - They just work together
+- **Factory fixtures** - Callables to create instances on-demand
+- **Plugin system** - Custom reporters, filters
+- **Last-failed mode** - Re-run only failed tests with `--lf`
+- **CTRF reports** - Standardized JSON for CI/CD integration
+- **Native LLM evals** - Scored cases, JSONL history, `apte eval` (see [evals docs](https://renaudcepre.github.io/apte/evals/))
+
+## Why Not pytest?
+
+|          | pytest                          | Apte                              |
+|----------|---------------------------------|--------------------------------------|
+| Fixtures | Implicit (by name)              | Explicit (`Use(fixture)`)            |
+| Params   | Hidden in fixture               | Visible in test (`From()` + factory) |
+| Async    | Plugin required                 | Native                               |
+| Parallel | Plugin required                 | Built-in                             |
+| Cycles   | Runtime error                   | Prevented at registration            |
+| Evals    | External (deepeval, pydantic-…) | Native (`apte eval`, JSONL history) |
+
+pytest has a large ecosystem and extensive community. Apte is an alternative if you
+prefer FastAPI-style explicit dependencies and native async in your tests.
+
+## Documentation
+
+Full API reference, guides, and examples: [renaudcepre.github.io/apte](https://renaudcepre.github.io/apte/)
+
+## License
+
+MIT

apte-0.3.0/README.md

@@ -0,0 +1,180 @@
+![Apte](assets/logo-with-text-animate.svg)
+
+[![CI](https://github.com/renaudcepre/apte/actions/workflows/ci.yml/badge.svg)](https://github.com/renaudcepre/apte/actions/workflows/ci.yml)
+[![codecov](https://codecov.io/gh/renaudcepre/apte/graph/badge.svg?token=V0MLGEE5UZ)](https://codecov.io/gh/renaudcepre/apte)
+[![docs](https://img.shields.io/badge/docs-GitHub%20Pages-blue)](https://renaudcepre.github.io/apte/)
+
+**Write your tests and your LLM evals in one async-first framework.**
+
+An eval is just a test that returns a value - scored, not asserted. Your evals get
+real fixtures, dependency injection and parallelism, and live right next to the
+tests they ship with. Python 3.10+, installs lean (Rich UI optional).
+
+---
+
+## Why Apte?
+
+### Explicit Injection (IDE-Ready)
+
+**Ctrl+Click works.** Your IDE knows every type. No guessing where fixtures come from.
+
+```python
+def test_user(db: Annotated[Database, Use(database)]): ...
+```
+
+### Native Async & Parallelism
+
+Tests run as coroutines on a single event loop. No plugin needed.
+
+```bash
+apte run tests:session -n 10
+```
+
+### Smart Tagging (Tag Propagation)
+
+Tag a fixture once, every test using it inherits the tag automatically.
+
+```python
+@fixture(tags=["database"])
+def db(): ...
+
+session.bind(db)
+
+@session.test()
+def test_users(repo: Annotated[Repo, Use(user_repo)]): ...  # Also tagged "database"
+```
+
+```bash
+apte run tests:session --no-tag database  # Skips ALL tests touching DB
+```
+
+### Infra vs Code Errors (Error ≠ Fail)
+
+```
+✗ test_create_user: AssertionError       # Your bug - TEST FAILED
+⚠ test_with_db: [FIXTURE] ConnectionError  # Infra issue - SETUP ERROR
+```
+
+### Typed Parameterization
+
+```python
+CODES = ForEach([200, 201])
+
+@session.test()
+def test_status(code: Annotated[int, From(CODES)]): ...
+```
+
+### Native LLM Evals
+
+Score model outputs alongside your tests - same fixtures, same parallelism, same `apte` CLI. Cases get pass/fail + numeric metrics, persisted to JSONL for run-over-run comparison.
+
+```python
+from typing import Annotated
+from apte import ForEach, From, ApteSession
+from apte.evals import EvalCase, EvalSuite
+from apte.evals.evaluators import contains_keywords
+
+session = ApteSession()
+chatbot_suite = EvalSuite("chatbot")
+session.add_suite(chatbot_suite)
+
+cases = ForEach([
+    EvalCase(name="capital_fr", inputs="Capital of France?", expected="Paris"),
+])
+
+@chatbot_suite.eval(evaluators=[contains_keywords(keywords=["paris"])])
+async def chatbot(case: Annotated[EvalCase, From(cases)]) -> str:
+    return await my_agent(case.inputs)  # your LLM call
+```
+
+```bash
+apte eval evals.session:session   # runs are recorded to .apte/history.jsonl
+```
+
+See [Evals docs](https://renaudcepre.github.io/apte/evals/) for evaluators, judges, and scoring.
+
+---
+
+## Quick Start
+
+```python
+from apte import ApteSession
+
+session = ApteSession()
+
+
+def inc(x):
+    return x + 1
+
+
+@session.test()
+def test_answer():
+    assert inc(3) == 4
+```
+
+```bash
+apte run test_sample:session
+```
+
+## Installation
+
+Apte is not yet on PyPI. Install directly from GitHub:
+
+```bash
+# With uv (recommended)
+uv add git+https://github.com/renaudcepre/apte.git
+
+# With pip
+pip install git+https://github.com/renaudcepre/apte.git
+```
+
+## CLI
+
+```bash
+apte run module:session                    # Run tests
+apte run module:session -n 4               # Parallel (4 workers)
+apte run module:session --lf               # Re-run failed tests only
+apte run module:session --collect-only     # List tests without running
+apte run module:session --cache-clear      # Clear cache before run
+apte run module:session --app-dir src      # Look for module in src/
+apte run module:session --ctrf-output r.json  # CTRF report for CI/CD
+
+apte eval module:session                   # Run LLM evals
+apte eval module:session --tag safety      # Filter by case tag
+apte eval module:session --last-failed     # Re-run failed cases only
+```
+
+## Features
+
+- **Explicit DI** - No guessing which fixture you're using
+- **Async native** - No plugin needed, just `async def`
+- **Parallel execution** - Built-in with `-n 4`
+- **Scoped fixtures** - `SESSION`, `SUITE`, `TEST`
+- **Mix sync/async** - They just work together
+- **Factory fixtures** - Callables to create instances on-demand
+- **Plugin system** - Custom reporters, filters
+- **Last-failed mode** - Re-run only failed tests with `--lf`
+- **CTRF reports** - Standardized JSON for CI/CD integration
+- **Native LLM evals** - Scored cases, JSONL history, `apte eval` (see [evals docs](https://renaudcepre.github.io/apte/evals/))
+
+## Why Not pytest?
+
+|          | pytest                          | Apte                              |
+|----------|---------------------------------|--------------------------------------|
+| Fixtures | Implicit (by name)              | Explicit (`Use(fixture)`)            |
+| Params   | Hidden in fixture               | Visible in test (`From()` + factory) |
+| Async    | Plugin required                 | Native                               |
+| Parallel | Plugin required                 | Built-in                             |
+| Cycles   | Runtime error                   | Prevented at registration            |
+| Evals    | External (deepeval, pydantic-…) | Native (`apte eval`, JSONL history) |
+
+pytest has a large ecosystem and extensive community. Apte is an alternative if you
+prefer FastAPI-style explicit dependencies and native async in your tests.
+
+## Documentation
+
+Full API reference, guides, and examples: [renaudcepre.github.io/apte](https://renaudcepre.github.io/apte/)
+
+## License
+
+MIT

apte-0.3.0/apte/__init__.py

@@ -0,0 +1,55 @@
+from apte import console
+from apte.api import collect_tests, list_tags, run_session
+from apte.assertions import ExceptionInfo, RaisesContext, raises, warns
+from apte.core.session import ApteSession
+from apte.core.suite import ApteSuite
+from apte.di.decorators import factory, fixture
+from apte.di.factory import FixtureFactory
+from apte.di.markers import ForEach, From, Use
+from apte.entities import FixtureCallable, Retry, Skip, Xfail
+from apte.exceptions import ApteError, CircularDependencyError, FixtureError
+from apte.fixtures.builtins import caplog, mocker, tmp_path
+from apte.fixtures.mocker import AsyncMockType, Mocker, MockType
+from apte.loader import LoadError, load_session
+from apte.plugin import PluginBase
+from apte.shell import CommandResult, Shell
+
+__version__ = "0.3.0"
+
+__all__ = [
+    "ApteError",
+    "ApteSession",
+    "ApteSuite",
+    "AsyncMockType",
+    "CircularDependencyError",
+    "CommandResult",
+    "ExceptionInfo",
+    "FixtureCallable",
+    "FixtureError",
+    "FixtureFactory",
+    "ForEach",
+    "From",
+    "LoadError",
+    "MockType",
+    "Mocker",
+    "PluginBase",
+    "RaisesContext",
+    "Retry",
+    "Shell",
+    "Skip",
+    "Use",
+    "Xfail",
+    "__version__",
+    "caplog",
+    "collect_tests",
+    "console",
+    "factory",
+    "fixture",
+    "list_tags",
+    "load_session",
+    "mocker",
+    "raises",
+    "run_session",
+    "tmp_path",
+    "warns",
+]

apte-0.3.0/apte/__main__.py

@@ -0,0 +1,5 @@
+"""Allow running apte as: python -m apte / coverage run -m apte."""
+
+from apte.cli.main import main
+
+main()

apte-0.3.0/apte/api.py

@@ -0,0 +1,194 @@
+"""Public API for running Apte sessions.
+
+This module provides the main entry points for running tests programmatically,
+independent of any CLI or adapter implementation.
+
+Example:
+    from apte import ApteSession
+    from apte.api import run_session
+
+    session = ApteSession()
+
+    @session.test()
+    def test_example():
+        assert True
+
+    success = run_session(session)
+"""
+
+from __future__ import annotations
+
+import asyncio
+from typing import TYPE_CHECKING
+
+from apte.core.collector import Collector
+from apte.core.runner import TestRunner
+from apte.core.suite import (
+    ApteSuite,  # noqa: TC001 - used at runtime in list_tags
+)
+from apte.events.types import Event
+from apte.filters.keyword import KeywordFilterPlugin
+from apte.filters.kind import KindFilterPlugin
+from apte.filters.suite import SuiteFilterPlugin
+from apte.plugin import PluginBase, PluginContext
+from apte.tags.plugin import TagFilterPlugin
+
+if TYPE_CHECKING:
+    from apte.core.session import ApteSession
+    from apte.entities import RunResult, TestItem
+
+
+def run_session(  # noqa: PLR0913 - public API with many optional params
+    session: ApteSession,
+    concurrency: int | None = None,
+    exitfirst: bool = False,
+    last_failed: bool = False,
+    cache_clear: bool = False,
+    include_tags: set[str] | None = None,
+    exclude_tags: set[str] | None = None,
+    capture: bool = True,
+    suite_filter: str | None = None,
+    keyword_patterns: list[str] | None = None,
+    log_file: bool = True,
+    force_no_color: bool = False,
+    *,
+    ctx: PluginContext | None = None,
+) -> RunResult:
+    """Run a test session and return result with success and interrupted status.
+
+    This is the main entry point for running tests programmatically.
+
+    Args:
+        session: The ApteSession to run.
+        concurrency: Number of concurrent workers (None = use session default).
+        exitfirst: Stop after first failure.
+        last_failed: Only run tests that failed in the last run.
+        cache_clear: Clear the cache before running.
+        include_tags: Only run tests with these tags (OR logic).
+        exclude_tags: Exclude tests with these tags.
+        capture: Capture stdout/stderr during tests (default: True).
+        suite_filter: Only run tests in this suite (::SuiteName syntax).
+        keyword_patterns: Only run tests matching these patterns (-k flag).
+        log_file: Write output to .apte/last_run.log (default: True).
+        force_no_color: Disable colors (--no-color flag).
+        ctx: Plugin context (if provided, overrides individual params above).
+
+    Returns:
+        RunResult with success status and interrupted flag.
+    """
+    # Apply session-level settings from ctx or params
+    if ctx is not None:
+        if ctx.get("concurrency") is not None:
+            session.concurrency = ctx.get("concurrency")
+        session.exitfirst = ctx.get("exitfirst", False)
+        session.capture = not ctx.get("no_capture", False)
+    else:
+        if concurrency is not None:
+            session.concurrency = concurrency
+        session.exitfirst = exitfirst
+        session.capture = capture
+
+    # Register default plugins if none registered
+    if not session.plugin_classes:
+        session.register_default_plugins()
+
+    # Build context from parameters if not provided
+    if ctx is None:
+        ctx = PluginContext(
+            args={
+                "last_failed": last_failed,
+                "cache_clear": cache_clear,
+                "tags": list(include_tags) if include_tags else [],
+                "exclude_tags": list(exclude_tags) if exclude_tags else [],
+                "target_suite": suite_filter,
+                "keywords": keyword_patterns or [],
+                "no_log_file": not log_file,
+                "no_color": force_no_color,
+            }
+        )
+
+    session.activate_plugins(ctx)
+
+    runner = TestRunner(session)
+    return runner.run()
+
+
+def collect_tests(  # noqa: PLR0913 - public API with many optional params
+    session: ApteSession,
+    include_tags: set[str] | None = None,
+    exclude_tags: set[str] | None = None,
+    suite_filter: str | None = None,
+    keyword_patterns: list[str] | None = None,
+    *,
+    ctx: PluginContext | None = None,
+) -> list[TestItem]:
+    """Collect tests from a session without running them.
+
+    Args:
+        session: The ApteSession to collect from.
+        include_tags: Only include tests with these tags.
+        exclude_tags: Exclude tests with these tags.
+        suite_filter: Only include tests in this suite.
+        keyword_patterns: Only include tests matching these patterns.
+        ctx: Plugin context (if provided, overrides individual params above).
+
+    Returns:
+        List of collected TestItem objects.
+    """
+    # Build context from parameters if not provided
+    if ctx is None:
+        ctx = PluginContext(
+            args={
+                "tags": list(include_tags) if include_tags else [],
+                "exclude_tags": list(exclude_tags) if exclude_tags else [],
+                "target_suite": suite_filter,
+                "keywords": keyword_patterns or [],
+            }
+        )
+
+    # Activate filter plugins
+    filter_plugins: list[type[PluginBase]] = [
+        TagFilterPlugin,
+        SuiteFilterPlugin,
+        KeywordFilterPlugin,
+        KindFilterPlugin,
+    ]
+    for plugin_class in filter_plugins:
+        instance = plugin_class.activate(ctx)
+        if instance is not None:
+            session.register_plugin(instance)
+
+    collector = Collector()
+    items = collector.collect(session)
+    return asyncio.run(session.events.emit_and_collect(Event.COLLECTION_FINISH, items))
+
+
+def list_tags(session: ApteSession) -> set[str]:
+    """List all declared tags in a session.
+
+    Args:
+        session: The ApteSession to inspect.
+
+    Returns:
+        Set of all tag names declared on fixtures, suites, and tests.
+    """
+    all_tags: set[str] = set()
+
+    for fixture_reg in session.fixtures:
+        all_tags.update(fixture_reg.tags)
+
+    for test_reg in session.tests:
+        all_tags.update(test_reg.tags)
+
+    def collect_from_suites(suites: list[ApteSuite]) -> None:
+        for suite in suites:
+            all_tags.update(suite.tags)
+            for fixture_reg in suite.fixtures:
+                all_tags.update(fixture_reg.tags)
+            for test_reg in suite.tests:
+                all_tags.update(test_reg.tags)
+            collect_from_suites(suite.suites)
+
+    collect_from_suites(session.suites)
+
+    return all_tags