pytest-markdown-docs 0.7.0__tar.gz → 0.8.0__tar.gz

{pytest_markdown_docs-0.7.0 → pytest_markdown_docs-0.8.0}/.github/workflows/ci.yml

@@ -6,7 +6,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
     steps:
       - uses: actions/checkout@v4
 

{pytest_markdown_docs-0.7.0 → pytest_markdown_docs-0.8.0}/.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: "v0.2.1"
+    rev: "v0.9.10"
     hooks:
       - id: ruff
         # Autofix, and respect `exclude` and `extend-exclude` settings.

{pytest_markdown_docs-0.7.0 → pytest_markdown_docs-0.8.0}/PKG-INFO

@@ -1,11 +1,12 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: pytest-markdown-docs
-Version: 0.7.0
+Version: 0.8.0
 Summary: Run markdown code fences through pytest
 Author: Modal Labs
 Author-email: Elias Freider <elias@modal.com>
-License: MIT
-Requires-Python: >=3.8
+License-Expression: MIT
+License-File: LICENSE
+Requires-Python: >=3.9
 Requires-Dist: markdown-it-py<4.0,>=2.2.0
 Requires-Dist: pytest>=7.0.0
 Description-Content-Type: text/markdown

pytest_markdown_docs-0.8.0/playground/conftest.py

@@ -0,0 +1,17 @@
+import time
+
+import pytest
+import pytest_asyncio
+import asyncio
+
+
+@pytest_asyncio.fixture()
+async def async_fix():
+    await asyncio.sleep(1)
+    yield "ahello"
+
+
+@pytest.fixture()
+def non_async_fix():
+    time.sleep(1)
+    yield "hello"

pytest_markdown_docs-0.8.0/playground/foo.md

@@ -0,0 +1,6 @@
+from playground.conftest import non_async_fix
+
+```py fixture:async_fix fixture:non_async_fix
+print(async_fix)
+print(non_async_fix)
+```

pytest_markdown_docs-0.8.0/playground/foo.py

@@ -0,0 +1,3 @@
+def test_foo(async_fix, non_async_fix):
+    print(async_fix)
+    print(non_async_fix)

{pytest_markdown_docs-0.7.0 → pytest_markdown_docs-0.8.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "pytest-markdown-docs"
-version = "0.7.0"
+version = "0.8.0"
 description = "Run markdown code fences through pytest"
 readme = "README.md"
 authors = [
@@ -8,7 +8,7 @@ authors = [
     { name = "Elias Freider", email = "elias@modal.com" } 
 ]
 license = "MIT"
-requires-python = ">=3.8"
+requires-python = ">=3.9"
 dependencies = [
     "markdown-it-py>=2.2.0,<4.0",
     "pytest>=7.0.0",
@@ -28,5 +28,5 @@ dev-dependencies = [
     "mypy>=1.12.1",
     "pre-commit>=3.5.0",
     "pytest~=8.1.0",
-    "ruff>=0.7.0",
+    "ruff~=0.9.10",
 ]

pytest_markdown_docs-0.8.0/src/pytest_markdown_docs/_runners.py

@@ -0,0 +1,133 @@
+import abc
+import ast
+import traceback
+import typing
+from abc import abstractmethod
+
+import pytest
+
+from pytest_markdown_docs.definitions import FenceTestDefinition
+
+_default_runner: typing.Optional["_Runner"] = None
+_registered_runners = {}
+
+
+class _Runner(metaclass=abc.ABCMeta):
+    @abstractmethod
+    def runtest(self, test: FenceTestDefinition, args: dict[str, typing.Any]): ...
+
+    @abstractmethod
+    def repr_failure(
+        self,
+        test: FenceTestDefinition,
+        excinfo: pytest.ExceptionInfo[BaseException],
+        style=None,
+    ): ...
+
+
+RUNNER_TYPE = typing.TypeVar("RUNNER_TYPE", bound=type[_Runner])
+
+
+def register_runner(*, default: bool = False):
+    """Decorator for adding custom runners
+
+    e.g.
+    @register_runner()
+    def my_runner(src):
+        exec(src)
+    """
+
+    def decorator(r: RUNNER_TYPE) -> RUNNER_TYPE:
+        global _default_runner
+        runner = r()
+        _registered_runners[r.__name__] = runner
+        if default:
+            _default_runner = runner
+        return r
+
+    return decorator
+
+
+@register_runner(default=True)
+class DefaultRunner(_Runner):
+    def runtest(self, test: FenceTestDefinition, args):
+        try:
+            tree = ast.parse(test.source, filename=test.source_path)
+        except SyntaxError:
+            raise
+
+        try:
+            # if we don't compile the code, it seems we get name lookup errors
+            # for functions etc. when doing cross-calls across inline functions
+            compiled = compile(
+                tree, filename=test.source_path, mode="exec", dont_inherit=True
+            )
+        except SyntaxError:
+            raise
+
+        exec(compiled, args)
+
+    def repr_failure(
+        self,
+        test: FenceTestDefinition,
+        excinfo: pytest.ExceptionInfo[BaseException],
+        style=None,
+    ):
+        """This renders a traceback starting at the stack from of the code fence
+
+        Also displays a line-numbered excerpt of the code fence that ran.
+        """
+
+        rawlines = test.source.rstrip("\n").split("\n")
+
+        # custom formatted traceback to translate line numbers and markdown files
+        traceback_lines = []
+        stack_summary = traceback.StackSummary.extract(traceback.walk_tb(excinfo.tb))
+        start_capture = False
+
+        start_line = test.start_line
+
+        for frame_summary in stack_summary:
+            if frame_summary.filename == str(test.source_path):
+                # start capturing frames the first time we enter user code
+                start_capture = True
+
+            if start_capture:
+                lineno = frame_summary.lineno
+                line = frame_summary.line or ""
+                linespec = f"line {lineno}"
+                traceback_lines.append(
+                    f"""  File "{frame_summary.filename}", {linespec}, in {frame_summary.name}"""
+                )
+                traceback_lines.append(f"    {line.lstrip()}")
+
+        maxdigits = len(str(len(rawlines)))
+        code_margin = "   "
+        numbered_code = "\n".join(
+            [
+                f"{i:>{maxdigits}}{code_margin}{line}"
+                for i, line in enumerate(rawlines[start_line:], start_line + 1)
+            ]
+        )
+
+        pretty_traceback = "\n".join(traceback_lines)
+        pt = f"""Traceback (most recent call last):
+{pretty_traceback}
+{excinfo.exconly()}"""
+
+        return f"""Error in code block:
+{maxdigits * " "}{code_margin}```
+{numbered_code}
+{maxdigits * " "}{code_margin}```
+{pt}
+"""
+
+
+def get_runner(name: typing.Optional[str]) -> _Runner:
+    if name is None:
+        assert _default_runner is not None
+        return _default_runner
+
+    if name not in _registered_runners:
+        raise Exception(f"No such pytest-markdown-docs runner: {name}")
+    return _registered_runners[name]

pytest_markdown_docs-0.8.0/src/pytest_markdown_docs/definitions.py

@@ -0,0 +1,19 @@
+import pathlib
+import typing
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class FenceTestDefinition:
+    source: str
+    fixture_names: typing.Sequence[str]
+    start_line: int
+    source_path: pathlib.Path
+    runner_name: typing.Optional[str]
+
+
+@dataclass(frozen=True)
+class ObjectTestDefinition:
+    intra_object_index: int
+    object_name: str
+    fence_test: FenceTestDefinition

{pytest_markdown_docs-0.7.0 → pytest_markdown_docs-0.8.0}/src/pytest_markdown_docs/plugin.py

@@ -1,9 +1,6 @@
-import ast
 import inspect
-import traceback
 import types
 import pathlib
-from dataclasses import dataclass
 
 import pytest
 import typing
@@ -15,7 +12,8 @@ from _pytest.pathlib import import_path
 import logging
 
 from pytest_markdown_docs import hooks
-
+from pytest_markdown_docs.definitions import FenceTestDefinition, ObjectTestDefinition
+from pytest_markdown_docs._runners import get_runner
 
 if pytest.version_tuple >= (8, 0, 0):
     from _pytest.fixtures import TopRequest
@@ -37,23 +35,12 @@ class FenceSyntax(Enum):
     superfences = "superfences"
 
 
-@dataclass
-class FenceTest:
-    source: str
-    fixture_names: typing.List[str]
-    start_line: int
-
-
-@dataclass
-class ObjectTest:
-    intra_object_index: int
-    object_name: str
-    fence_test: FenceTest
-
-
 def get_docstring_start_line(obj) -> typing.Optional[int]:
     # Get the source lines and the starting line number of the object
-    source_lines, start_line = inspect.getsourcelines(obj)
+    try:
+        source_lines, start_line = inspect.getsourcelines(obj)
+    except OSError:
+        return None
 
     # Find the line in the source code that starts with triple quotes (""" or ''')
     for idx, line in enumerate(source_lines):
@@ -69,18 +56,18 @@ class MarkdownInlinePythonItem(pytest.Item):
         self,
         name: str,
         parent: typing.Union["MarkdownDocstringCodeModule", "MarkdownTextFile"],
-        code: str,
-        fixture_names: typing.List[str],
-        start_line: int,
+        test_definition: FenceTestDefinition,
     ) -> None:
         super().__init__(name, parent)
         self.add_marker(MARKER_NAME)
-        self.code = code
+        self.code = test_definition.source
         self.obj = None
-        self.user_properties.append(("code", code))
-        self.start_line = start_line
-        self.fixturenames = fixture_names
+        self.test_definition = test_definition
+        self.user_properties.append(("code", test_definition.source))
+        self.start_line = test_definition.start_line
+        self.fixturenames = test_definition.fixture_names
         self.nofuncargs = True
+        self.runner_name = test_definition.runner_name
 
     def setup(self):
         def func() -> None:
@@ -92,6 +79,7 @@ class MarkdownInlinePythonItem(pytest.Item):
         )
         self.fixture_request = TopRequest(self, _ispytest=True)
         self.fixture_request._fillfixtures()
+        self.runner = get_runner(self.runner_name)
 
     def runtest(self):
         global_sets = self.parent.config.hook.pytest_markdown_docs_globals()
@@ -111,79 +99,36 @@ class MarkdownInlinePythonItem(pytest.Item):
         for argname, value in self.funcargs.items():
             all_globals[argname] = value
 
-        try:
-            tree = ast.parse(self.code, filename=self.path)
-        except SyntaxError:
-            raise
-
-        try:
-            # if we don't compile the code, it seems we get name lookup errors
-            # for functions etc. when doing cross-calls across inline functions
-            compiled = compile(tree, filename=self.path, mode="exec", dont_inherit=True)
-        except SyntaxError:
-            raise
-
-        exec(compiled, all_globals)
+        # this ensures that pytest's stdout/stderr capture works during the test:
+        capman = self.config.pluginmanager.getplugin("capturemanager")
+        with capman.global_and_fixture_disabled():
+            self.runner.runtest(self.test_definition, all_globals)
 
     def repr_failure(
         self,
         excinfo: ExceptionInfo[BaseException],
         style=None,
-    ):
-        rawlines = self.code.rstrip("\n").split("\n")
-
-        # custom formatted traceback to translate line numbers and markdown files
-        traceback_lines = []
-        stack_summary = traceback.StackSummary.extract(traceback.walk_tb(excinfo.tb))
-        start_capture = False
-
-        start_line = self.start_line
-
-        for frame_summary in stack_summary:
-            if frame_summary.filename == str(self.path):
-                # start capturing frames the first time we enter user code
-                start_capture = True
-
-            if start_capture:
-                lineno = frame_summary.lineno
-                line = frame_summary.line or ""
-                linespec = f"line {lineno}"
-                traceback_lines.append(
-                    f"""  File "{frame_summary.filename}", {linespec}, in {frame_summary.name}"""
-                )
-                traceback_lines.append(f"    {line.lstrip()}")
-
-        maxdigits = len(str(len(rawlines)))
-        code_margin = "   "
-        numbered_code = "\n".join(
-            [
-                f"{i:>{maxdigits}}{code_margin}{line}"
-                for i, line in enumerate(rawlines[start_line:], start_line + 1)
-            ]
-        )
+    ) -> str:
+        return self.runner.repr_failure(self.test_definition, excinfo, style)
 
-        pretty_traceback = "\n".join(traceback_lines)
-        pt = f"""Traceback (most recent call last):
-{pretty_traceback}
-{excinfo.exconly()}"""
+    def reportinfo(self):
+        return self.path, self.start_line, self.name
 
-        return f"""Error in code block:
-{maxdigits * " "}{code_margin}```
-{numbered_code}
-{maxdigits * " "}{code_margin}```
-{pt}
-"""
 
-    def reportinfo(self):
-        return self.name, 0, self.nodeid
+def get_prefixed_strings(
+    seq: typing.Collection[str], prefix: str
+) -> typing.Sequence[str]:
+    # return strings matching a prefix, with the prefix stripped
+    return tuple(s[len(prefix) :] for s in seq if s.startswith(prefix))
 
 
 def extract_fence_tests(
     markdown_string: str,
     start_line_offset: int,
+    source_path: pathlib.Path,
     markdown_type: str = "md",
     fence_syntax: FenceSyntax = FenceSyntax.default,
-) -> typing.Generator[FenceTest, None, None]:
+) -> typing.Generator[FenceTestDefinition, None, None]:
     import markdown_it
 
     mi = markdown_it.MarkdownIt(config="commonmark")
@@ -226,10 +171,23 @@ def extract_fence_tests(
             add_blank_lines = start_line - prev.count("\n")
             code_block = prev + ("\n" * add_blank_lines) + block.content
 
-            fixture_names = [
-                f[len("fixture:") :] for f in code_options if f.startswith("fixture:")
-            ]
-            yield FenceTest(code_block, fixture_names, start_line)
+            fixture_names = get_prefixed_strings(code_options, "fixture:")
+            runner_names = get_prefixed_strings(code_options, "runner:")
+            if len(runner_names) == 0:
+                runner_name = None
+            elif len(runner_names) > 1:
+                raise Exception(
+                    f"Multiple runners are not supported, use a single one instead: {runner_names}"
+                )
+            else:
+                runner_name = runner_names[0]
+            yield FenceTestDefinition(
+                code_block,
+                fixture_names,
+                start_line,
+                source_path=source_path,
+                runner_name=runner_name,
+            )
             prev = code_block
 
 
@@ -291,50 +249,67 @@ class MarkdownDocstringCodeModule(pytest.Module):
             # but unsupported before pytest 8.1...
             module = import_path(self.path, root=self.config.rootpath)
 
-        for object_test in self.find_object_tests_recursive(module.__name__, module):
+        for object_test in self.find_object_tests_recursive(
+            module.__name__, module, set(), set()
+        ):
             fence_test = object_test.fence_test
             yield MarkdownInlinePythonItem.from_parent(
                 self,
-                name=f"{object_test.object_name}[CodeFence#{object_test.intra_object_index+1}][line:{fence_test.start_line}]",
-                code=fence_test.source,
-                fixture_names=fence_test.fixture_names,
-                start_line=fence_test.start_line,
+                name=f"{object_test.object_name}[CodeFence#{object_test.intra_object_index + 1}][line:{fence_test.start_line}]",
+                test_definition=fence_test,
             )
 
     def find_object_tests_recursive(
-        self, module_name: str, object: typing.Any
-    ) -> typing.Generator[ObjectTest, None, None]:
+        self,
+        module_name: str,
+        object: typing.Any,
+        _visited_objects: typing.Set[int],
+        _found_tests: typing.Set[typing.Tuple[str, int]],
+    ) -> typing.Generator[ObjectTestDefinition, None, None]:
+        if id(object) in _visited_objects:
+            return
+        _visited_objects.add(id(object))
         docstr = inspect.getdoc(object)
 
-        if docstr:
-            docstring_offset = get_docstring_start_line(object)
-            if docstring_offset is None:
-                logger.warning(
-                    "Could not find line number offset for docstring: {docstr}"
-                )
-                docstring_offset = 0
-
-            obj_name = (
-                getattr(object, "__qualname__", None)
-                or getattr(object, "__name__", None)
-                or "<Unnamed obj>"
-            )
-            fence_syntax = FenceSyntax(self.config.option.markdowndocs_syntax)
-            for i, fence_test in enumerate(
-                extract_fence_tests(docstr, docstring_offset, fence_syntax=fence_syntax)
-            ):
-                yield ObjectTest(i, obj_name, fence_test)
-
         for member_name, member in inspect.getmembers(object):
-            if member_name.startswith("_"):
-                continue
-
             if (
                 inspect.isclass(member)
                 or inspect.isfunction(member)
                 or inspect.ismethod(member)
             ) and member.__module__ == module_name:
-                yield from self.find_object_tests_recursive(module_name, member)
+                yield from self.find_object_tests_recursive(
+                    module_name, member, _visited_objects, _found_tests
+                )
+
+        if docstr:
+            docstring_offset = get_docstring_start_line(object)
+            if docstring_offset is None:
+                logger.warning(
+                    f"Could not find line number offset for docstring: {docstr}"
+                )
+            else:
+                obj_name = (
+                    getattr(object, "__qualname__", None)
+                    or getattr(object, "__name__", None)
+                    or "<Unnamed obj>"
+                )
+                fence_syntax = FenceSyntax(self.config.option.markdowndocs_syntax)
+                for i, fence_test in enumerate(
+                    extract_fence_tests(
+                        docstr,
+                        docstring_offset,
+                        source_path=self.path,
+                        fence_syntax=fence_syntax,
+                    )
+                ):
+                    found_test = ObjectTestDefinition(i, obj_name, fence_test)
+                    found_test_location = (
+                        module_name,
+                        found_test.fence_test.start_line,
+                    )
+                    if found_test_location not in _found_tests:
+                        _found_tests.add(found_test_location)
+                        yield found_test
 
 
 class MarkdownTextFile(pytest.File):
@@ -345,6 +320,7 @@ class MarkdownTextFile(pytest.File):
         for i, fence_test in enumerate(
             extract_fence_tests(
                 markdown_content,
+                source_path=self.path,
                 start_line_offset=0,
                 markdown_type=self.path.suffix.replace(".", ""),
                 fence_syntax=fence_syntax,
@@ -352,10 +328,8 @@ class MarkdownTextFile(pytest.File):
         ):
             yield MarkdownInlinePythonItem.from_parent(
                 self,
-                name=f"[CodeFence#{i+1}][line:{fence_test.start_line}]",
-                code=fence_test.source,
-                fixture_names=fence_test.fixture_names,
-                start_line=fence_test.start_line,
+                name=f"[CodeFence#{i + 1}][line:{fence_test.start_line}]",
+                test_definition=fence_test,
             )
 
 

{pytest_markdown_docs-0.7.0 → pytest_markdown_docs-0.8.0}/tests/plugin_test.py

@@ -142,9 +142,9 @@ Traceback \(most recent call last\):
 Exception: doh
 """.strip()
     pytest_output = "\n".join(line.rstrip() for line in result.outlines).strip()
-    assert (
-        re.search(expected_output_pattern, pytest_output) is not None
-    ), "Output traceback doesn't match expected value"
+    assert re.search(expected_output_pattern, pytest_output) is not None, (
+        "Output traceback doesn't match expected value"
+    )
 
 
 def test_autouse_fixtures(testdir):
@@ -431,3 +431,42 @@ def test_error_origin_before_docstring_traceback(testdir, support_dir):
         ],
         consecutive=True,
     )
+
+
+def test_custom_runner(testdir):
+    testdir.makeconftest(
+        """
+        import pytest_markdown_docs._runners
+        
+        @pytest_markdown_docs._runners.register_runner()
+        class LinesAreAllFoo(pytest_markdown_docs._runners.DefaultRunner):
+            def runtest(self, test, args):
+                lines = test.source.strip().split("\\n")
+                for line in lines:
+                    assert line == "foo"
+    """
+    )
+    testdir.makefile(
+        ".md",
+        """
+        ```python runner:LinesAreAllFoo
+        foo
+        foo
+        ```
+        
+        ```python runner:LinesAreAllFoo
+        foo
+        bar
+        ```
+    """,
+    )
+
+    result = testdir.runpytest("-v", "--markdown-docs")
+    result.assert_outcomes(passed=1, failed=1)
+    result.stdout.re_match_lines(
+        [
+            r".*\[CodeFence#1\]\[line:1\].*PASSED.*",
+            r".*\[CodeFence#2\]\[line:6\].*FAILED.*",
+        ],
+        consecutive=True,
+    )