passagemath-repl 10.4.62__py3-none-any.whl

sage/doctest/sources.py

@@ -0,0 +1,1657 @@
+# sage_setup: distribution = sagemath-repl
+"""
+Classes for sources of doctests
+
+This module defines various classes for sources from which doctests
+originate, such as files, functions or database entries.
+
+AUTHORS:
+
+- David Roe (2012-03-27) -- initial version, based on Robert Bradshaw's code.
+"""
+
+# ****************************************************************************
+#       Copyright (C) 2012-2013 David Roe <roed.math@gmail.com>
+#                     2012      Robert Bradshaw <robertwb@gmail.com>
+#                     2012      William Stein <wstein@gmail.com>
+#                     2013      R. Andrew Ohana
+#                     2013-2017 Jeroen Demeyer <jdemeyer@cage.ugent.be>
+#                     2013-2019 John H. Palmieri
+#                     2014      Volker Braun
+#                     2014-2022 Frédéric Chapoton
+#                     2017      Erik M. Bray
+#                     2021      Sébastien Labbé
+#                     2021-2023 Matthias Koeppe
+#
+#  Distributed under the terms of the GNU General Public License (GPL)
+#
+#                  https://www.gnu.org/licenses/
+# ****************************************************************************
+
+import os
+import sys
+import re
+import random
+import doctest
+from sage.cpython.string import bytes_to_str
+from sage.repl.load import load
+from sage.misc.lazy_attribute import lazy_attribute
+from sage.misc.package_dir import is_package_or_sage_namespace_package_dir
+from .parsing import SageDocTestParser
+from .util import NestedName
+from sage.structure.dynamic_class import dynamic_class
+from sage.env import SAGE_SRC, SAGE_LIB
+
+# Python file parsing
+triple_quotes = re.compile(r"\s*[rRuU]*((''')|(\"\"\"))")
+name_regex = re.compile(r".*\s(\w+)([(].*)?:")
+
+# LaTeX file parsing
+begin_verb = re.compile(r"\s*\\begin{verbatim}")
+end_verb = re.compile(r"\s*\\end{verbatim}\s*(%link)?")
+begin_lstli = re.compile(r"\s*\\begin{lstlisting}")
+end_lstli = re.compile(r"\s*\\end{lstlisting}\s*(%link)?")
+skip = re.compile(r".*%skip.*")
+
+# ReST file parsing
+link_all = re.compile(r"^\s*\.\.\s+linkall\s*$")
+double_colon = re.compile(r"^(\s*).*::\s*$")
+code_block = re.compile(r"^(\s*)[.][.]\s*code-block\s*::.*$")
+
+whitespace = re.compile(r"\s*")
+bitness_marker = re.compile('#.*(32|64)-bit')
+bitness_value = '64' if sys.maxsize > (1 << 32) else '32'
+
+# For neutralizing doctests
+find_prompt = re.compile(r"^(\s*)(>>>|sage:)(.*)")
+
+# For testing that enough doctests are created
+sagestart = re.compile(r"^(\s*(>>> |sage: ))\s*[^#\s]")
+untested = re.compile("(not implemented|not tested)")
+
+# For parsing a PEP 0263 encoding declaration
+pep_0263 = re.compile(br'^[ \t\v]*#.*?coding[:=]\s*([-\w.]+)')
+
+# Source line number in warning output
+doctest_line_number = re.compile(r"^\s*doctest:[0-9]")
+
+
+def get_basename(path):
+    """
+    This function returns the basename of the given path, e.g.
+    ``sage.doctest.sources`` or ``doc.ru.tutorial.tour_advanced``.
+
+    EXAMPLES::
+
+        sage: from sage.doctest.sources import get_basename
+        sage: import os
+        sage: get_basename(sage.doctest.sources.__file__)
+        'sage.doctest.sources'
+        sage: get_basename(os.path.join(sage.structure.__path__[0], 'element.pxd'))
+        'sage.structure.element.pxd'
+    """
+    if path is None:
+        return None
+    if not os.path.exists(path):
+        return path
+    path = os.path.abspath(path)
+    root = os.path.dirname(path)
+    # If the file is in the sage library, we can use our knowledge of
+    # the directory structure
+    dev = SAGE_SRC
+    sp = SAGE_LIB
+    if path.startswith(dev):
+        # there will be a branch name
+        i = path.find(os.path.sep, len(dev))
+        if i == -1:
+            # this source is the whole library....
+            return path
+        root = path[:i]
+    elif path.startswith(sp):
+        root = path[:len(sp)]
+    else:
+        # If this file is in some python package we can see how deep
+        # it goes.
+        while is_package_or_sage_namespace_package_dir(root):
+            root = os.path.dirname(root)
+    fully_qualified_path, ext = os.path.splitext(path[len(root) + 1:])
+    if os.path.split(path)[1] == '__init__.py':
+        fully_qualified_path = fully_qualified_path[:-9]
+    basename = fully_qualified_path.replace(os.path.sep, '.')
+    if ext in ['.pxd', '.pxi']:
+        # disambiguate from .pyx with the same basename
+        basename += ext
+    return basename
+
+
+class DocTestSource:
+    """
+    This class provides a common base class for different sources of doctests.
+
+    INPUT:
+
+    - ``options`` -- a :class:`sage.doctest.control.DocTestDefaults`
+      instance or equivalent
+    """
+    def __init__(self, options):
+        """
+        Initialization.
+
+        EXAMPLES::
+
+            sage: from sage.doctest.control import DocTestDefaults
+            sage: from sage.doctest.sources import FileDocTestSource
+            sage: filename = sage.doctest.sources.__file__
+            sage: FDS = FileDocTestSource(filename, DocTestDefaults())
+            sage: TestSuite(FDS).run()
+        """
+        self.options = options
+
+    def __eq__(self, other):
+        """
+        Comparison is just by comparison of attributes.
+
+        EXAMPLES::
+
+            sage: from sage.doctest.control import DocTestDefaults
+            sage: from sage.doctest.sources import FileDocTestSource
+            sage: filename = sage.doctest.sources.__file__
+            sage: DD = DocTestDefaults()
+            sage: FDS = FileDocTestSource(filename, DD)
+            sage: FDS2 = FileDocTestSource(filename, DD)
+            sage: FDS == FDS2
+            True
+        """
+        if type(self) is not type(other):
+            return False
+        return self.__dict__ == other.__dict__
+
+    def __ne__(self, other):
+        """
+        Test for non-equality.
+
+        EXAMPLES::
+
+            sage: from sage.doctest.control import DocTestDefaults
+            sage: from sage.doctest.sources import FileDocTestSource
+            sage: filename = sage.doctest.sources.__file__
+            sage: DD = DocTestDefaults()
+            sage: FDS = FileDocTestSource(filename, DD)
+            sage: FDS2 = FileDocTestSource(filename, DD)
+            sage: FDS != FDS2
+            False
+        """
+        return not (self == other)
+
+    def _process_doc(self, doctests, doc, namespace, start):
+        """
+        Appends doctests defined in ``doc`` to the list ``doctests``.
+
+        This function is called when a docstring block is completed
+        (either by ending a triple quoted string in a Python file,
+        unindenting from a comment block in a ReST file, or ending a
+        verbatim or lstlisting environment in a LaTeX file).
+
+        INPUT:
+
+        - ``doctests`` -- a running list of doctests to which the new
+          test(s) will be appended
+
+        - ``doc`` -- list of lines of a docstring, each including
+          the trailing newline
+
+        - ``namespace`` -- dictionary or
+          :class:`sage.doctest.util.RecordingDict`, used in the
+          creation of new :class:`doctest.DocTest` s
+
+        - ``start`` -- integer giving the line number of the start
+          of this docstring in the larger file
+
+        EXAMPLES::
+
+            sage: from sage.doctest.control import DocTestDefaults
+            sage: from sage.doctest.sources import FileDocTestSource
+            sage: from sage.doctest.parsing import SageDocTestParser
+            sage: filename = sage.doctest.util.__file__
+            sage: FDS = FileDocTestSource(filename, DocTestDefaults())
+            sage: doctests, _ = FDS.create_doctests({})
+            sage: manual_doctests = []
+            sage: for dt in doctests:
+            ....:     FDS.qualified_name = dt.name
+            ....:     FDS._process_doc(manual_doctests, dt.docstring, {}, dt.lineno-1)
+            sage: doctests == manual_doctests
+            True
+        """
+        docstring = "".join(doc)
+        new_doctests = self.parse_docstring(docstring, namespace, start)
+        sig_on_count_doc_doctest = "sig_on_count() # check sig_on/off pairings (virtual doctest)\n"
+        for dt in new_doctests:
+            if len(dt.examples) > 0 and not (hasattr(dt.examples[-1], 'sage_source')
+                                             and dt.examples[-1].sage_source == sig_on_count_doc_doctest):
+                # Line number refers to the end of the docstring
+                sigon = doctest.Example(sig_on_count_doc_doctest, "0\n", lineno=docstring.count("\n"))
+                sigon.sage_source = sig_on_count_doc_doctest
+                sigon.optional_tags = frozenset(self.file_optional_tags)
+                sigon.probed_tags = frozenset()
+                dt.examples.append(sigon)
+            doctests.append(dt)
+
+    @lazy_attribute
+    def file_optional_tags(self):
+        r"""
+        Return the set of tags that should apply to all doctests in this source.
+
+        This default implementation just returns the empty set.
+
+        EXAMPLES::
+
+            sage: from sage.doctest.control import DocTestDefaults
+            sage: from sage.doctest.sources import StringDocTestSource, PythonSource
+            sage: from sage.structure.dynamic_class import dynamic_class
+            sage: s = "'''\n    sage: 2 + 2\n    4\n'''"
+            sage: PythonStringSource = dynamic_class('PythonStringSource', (StringDocTestSource, PythonSource))
+            sage: PSS = PythonStringSource('<runtime>', s, DocTestDefaults(), 'runtime')
+            sage: PSS.file_optional_tags
+            set()
+        """
+        return set()
+
+    def _create_doctests(self, namespace, tab_okay=None):
+        """
+        Create a list of doctests defined in this source.
+
+        This function collects functionality common to file and string
+        sources, and is called by
+        :meth:`FileDocTestSource.create_doctests`.
+
+        INPUT:
+
+        - ``namespace`` -- dictionary or
+          :class:`sage.doctest.util.RecordingDict`, used in the
+          creation of new :class:`doctest.DocTest` s.
+
+        - ``tab_okay`` -- whether tabs are allowed in this source
+
+        OUTPUT:
+
+        - ``doctests`` -- list of doctests defined by this source
+
+        - ``extras`` -- dictionary with ``extras['tab']`` either
+          ``False`` or a list of linenumbers on which tabs appear
+
+        EXAMPLES::
+
+            sage: from sage.doctest.control import DocTestDefaults
+            sage: from sage.doctest.sources import FileDocTestSource
+            sage: from sage.doctest.util import NestedName
+            sage: filename = sage.doctest.sources.__file__
+            sage: FDS = FileDocTestSource(filename, DocTestDefaults())
+            sage: FDS.qualified_name = NestedName('sage.doctest.sources')
+            sage: doctests, extras = FDS._create_doctests({})
+            sage: len(doctests)
+            43
+            sage: extras['tab']
+            False
+            sage: extras['line_number']
+            False
+        """
+        if tab_okay is None:
+            tab_okay = isinstance(self, TexSource)
+        self._init()
+        self.line_shift = 0
+        self.parser = SageDocTestParser(self.options.optional,
+                                        self.options.long,
+                                        probed_tags=self.options.probe,
+                                        file_optional_tags=self.file_optional_tags)
+        self.linking = False
+        doctests = []
+        in_docstring = False
+        unparsed_doc = False
+        doc = []
+        start = None
+        tab_locations = []
+        contains_line_number = False
+        for lineno, line in self:
+            if doctest_line_number.search(line) is not None:
+                contains_line_number = True
+            if "\t" in line:
+                tab_locations.append(str(lineno+1))
+            if "SAGE_DOCTEST_ALLOW_TABS" in line:
+                tab_okay = True
+            just_finished = False
+            if in_docstring:
+                if self.ending_docstring(line):
+                    in_docstring = False
+                    just_finished = True
+                    self._process_doc(doctests, doc, namespace, start)
+                    unparsed_doc = False
+                else:
+                    bitness = bitness_marker.search(line)
+                    if bitness:
+                        if bitness.groups()[0] != bitness_value:
+                            self.line_shift += 1
+                            continue
+                        else:
+                            line = line[:bitness.start()] + "\n"
+                    if self.line_shift and (m := sagestart.match(line)):
+                        # We insert empty doctest lines to make up for the removed lines
+                        indent_and_prompt = m.group(1)
+                        doc.extend([indent_and_prompt + "# inserted to compensate for removed conditional doctest output\n"]
+                                   * self.line_shift)
+                        self.line_shift = 0
+                    doc.append(line)
+                    unparsed_doc = True
+            if not in_docstring and (not just_finished or self.start_finish_can_overlap):
+                # to get line numbers in linked docstrings correct we
+                # append a blank line to the doc list.
+                doc.append("\n")
+                if not line.strip():
+                    continue
+                if self.starting_docstring(line):
+                    in_docstring = True
+                    if self.linking:
+                        # If there's already a doctest, we overwrite it.
+                        if len(doctests) > 0:
+                            doctests.pop()
+                        if start is None:
+                            start = lineno
+                            doc = []
+                    else:
+                        self.line_shift = 0
+                        start = lineno
+                        doc = []
+        # In ReST files we can end the file without decreasing the indentation level.
+        if unparsed_doc:
+            self._process_doc(doctests, doc, namespace, start)
+
+        extras = {"tab": not tab_okay and tab_locations,
+                  "line_number": contains_line_number,
+                  "optionals": self.parser.optionals}
+        if self.options.randorder is not None and self.options.randorder is not False:
+            # we want to randomize even when self.randorder = 0
+            random.seed(self.options.randorder)
+            randomized = []
+            while doctests:
+                i = random.randint(0, len(doctests) - 1)
+                randomized.append(doctests.pop(i))
+            return randomized, extras
+        else:
+            return doctests, extras
+
+
+class StringDocTestSource(DocTestSource):
+    r"""
+    This class creates doctests from a string.
+
+    INPUT:
+
+    - ``basename`` -- string such as 'sage.doctests.sources', going
+      into the names of created doctests and examples
+
+    - ``source`` -- string, giving the source code to be parsed for
+      doctests
+
+    - ``options`` -- a :class:`sage.doctest.control.DocTestDefaults`
+      or equivalent
+
+    - ``printpath`` -- string, to be used in place of a filename
+      when doctest failures are displayed
+
+    - ``lineno_shift`` -- integer (default: 0) by which to shift
+      the line numbers of all doctests defined in this string
+
+    EXAMPLES::
+
+        sage: from sage.doctest.control import DocTestDefaults
+        sage: from sage.doctest.sources import StringDocTestSource, PythonSource
+        sage: from sage.structure.dynamic_class import dynamic_class
+        sage: s = "'''\n    sage: 2 + 2\n    4\n'''"
+        sage: PythonStringSource = dynamic_class('PythonStringSource',(StringDocTestSource, PythonSource))
+        sage: PSS = PythonStringSource('<runtime>', s, DocTestDefaults(), 'runtime')
+        sage: dt, extras = PSS.create_doctests({})
+        sage: len(dt)
+        1
+        sage: extras['tab']
+        []
+        sage: extras['line_number']
+        False
+
+        sage: s = "'''\n\tsage: 2 + 2\n\t4\n'''"
+        sage: PSS = PythonStringSource('<runtime>', s, DocTestDefaults(), 'runtime')
+        sage: dt, extras = PSS.create_doctests({})
+        sage: extras['tab']
+        ['2', '3']
+
+        sage: s = "'''\n    sage: import warnings; warnings.warn('foo')\n    doctest:1: UserWarning: foo \n'''"
+        sage: PSS = PythonStringSource('<runtime>', s, DocTestDefaults(), 'runtime')
+        sage: dt, extras = PSS.create_doctests({})
+        sage: extras['line_number']
+        True
+    """
+    def __init__(self, basename, source, options, printpath, lineno_shift=0):
+        r"""
+        Initialization.
+
+        TESTS::
+
+            sage: from sage.doctest.control import DocTestDefaults
+            sage: from sage.doctest.sources import StringDocTestSource, PythonSource
+            sage: from sage.structure.dynamic_class import dynamic_class
+            sage: s = "'''\n    sage: 2 + 2\n    4\n'''"
+            sage: PythonStringSource = dynamic_class('PythonStringSource',(StringDocTestSource, PythonSource))
+            sage: PSS = PythonStringSource('<runtime>', s, DocTestDefaults(), 'runtime')
+            sage: TestSuite(PSS).run()
+        """
+        self.qualified_name = NestedName(basename)
+        self.printpath = printpath
+        self.source = source
+        self.lineno_shift = lineno_shift
+        DocTestSource.__init__(self, options)
+
+    def __iter__(self):
+        r"""
+        Iterating over this source yields pairs ``(lineno, line)``.
+
+        EXAMPLES::
+
+            sage: from sage.doctest.control import DocTestDefaults
+            sage: from sage.doctest.sources import StringDocTestSource, PythonSource
+            sage: from sage.structure.dynamic_class import dynamic_class
+            sage: s = "'''\n    sage: 2 + 2\n    4\n'''"
+            sage: PythonStringSource = dynamic_class('PythonStringSource',(StringDocTestSource, PythonSource))
+            sage: PSS = PythonStringSource('<runtime>', s, DocTestDefaults(), 'runtime')
+            sage: for n, line in PSS:
+            ....:     print("{} {}".format(n, line))
+            0 '''
+            1     sage: 2 + 2
+            2     4
+            3 '''
+        """
+        for lineno, line in enumerate(self.source.split('\n')):
+            yield lineno + self.lineno_shift, line + '\n'
+
+    def create_doctests(self, namespace):
+        r"""
+        Create doctests from this string.
+
+        INPUT:
+
+        - ``namespace`` -- dictionary or :class:`sage.doctest.util.RecordingDict`
+
+        OUTPUT:
+
+        - ``doctests`` -- list of doctests defined by this string
+
+        - ``tab_locations`` -- either ``False`` or a list of linenumbers
+          on which tabs appear
+
+        EXAMPLES::
+
+            sage: from sage.doctest.control import DocTestDefaults
+            sage: from sage.doctest.sources import StringDocTestSource, PythonSource
+            sage: from sage.structure.dynamic_class import dynamic_class
+            sage: s = "'''\n    sage: 2 + 2\n    4\n'''"
+            sage: PythonStringSource = dynamic_class('PythonStringSource',(StringDocTestSource, PythonSource))
+            sage: PSS = PythonStringSource('<runtime>', s, DocTestDefaults(), 'runtime')
+            sage: dt, tabs = PSS.create_doctests({})
+            sage: for t in dt:
+            ....:     print("{} {}".format(t.name, t.examples[0].sage_source))
+            <runtime> 2 + 2
+        """
+        return self._create_doctests(namespace)
+
+
+class FileDocTestSource(DocTestSource):
+    """
+    This class creates doctests from a file.
+
+    INPUT:
+
+    - ``path`` -- string; the filename
+
+    - ``options`` -- a :class:`sage.doctest.control.DocTestDefaults`
+      instance or equivalent
+
+    EXAMPLES::
+
+        sage: from sage.doctest.control import DocTestDefaults
+        sage: from sage.doctest.sources import FileDocTestSource
+        sage: filename = sage.doctest.sources.__file__
+        sage: FDS = FileDocTestSource(filename, DocTestDefaults())
+        sage: FDS.basename
+        'sage.doctest.sources'
+
+    TESTS::
+
+        sage: TestSuite(FDS).run()
+
+    ::
+
+        sage: from sage.doctest.control import DocTestDefaults
+        sage: from sage.doctest.sources import FileDocTestSource
+        sage: filename = tmp_filename(ext='.txtt')
+        sage: FDS = FileDocTestSource(filename, DocTestDefaults())
+        Traceback (most recent call last):
+        ...
+        ValueError: unknown extension for the file to test (=...txtt),
+        valid extensions are: .py, .pyx, .pxd, .pxi, .sage, .spyx, .tex, .rst, .rst.txt
+    """
+    def __init__(self, path, options):
+        """
+        Initialization.
+
+        EXAMPLES::
+
+            sage: from sage.doctest.control import DocTestDefaults
+            sage: from sage.doctest.sources import FileDocTestSource
+            sage: filename = sage.doctest.sources.__file__
+            sage: FDS = FileDocTestSource(filename, DocTestDefaults(randorder=0))
+            sage: FDS.options.randorder
+            0
+        """
+        self.path = path
+        DocTestSource.__init__(self, options)
+        if path.endswith('.rst.txt'):
+            ext = '.rst.txt'
+        else:
+            base, ext = os.path.splitext(path)
+        valid_code_ext = ('.py', '.pyx', '.pxd', '.pxi', '.sage', '.spyx')
+        if ext in valid_code_ext:
+            self.__class__ = dynamic_class('PythonFileSource', (FileDocTestSource, PythonSource))
+            self.encoding = "utf-8"
+        elif ext == '.tex':
+            self.__class__ = dynamic_class('TexFileSource', (FileDocTestSource, TexSource))
+            self.encoding = "utf-8"
+        elif ext == '.rst' or ext == '.rst.txt':
+            self.__class__ = dynamic_class('RestFileSource', (FileDocTestSource, RestSource))
+            self.encoding = "utf-8"
+        else:
+            valid_ext = ", ".join(valid_code_ext + ('.tex', '.rst', '.rst.txt'))
+            raise ValueError("unknown extension for the file to test (={}),"
+                    " valid extensions are: {}".format(path, valid_ext))
+
+    def __iter__(self):
+        r"""
+        Iterating over this source yields pairs ``(lineno, line)``.
+
+        EXAMPLES::
+
+            sage: from sage.doctest.control import DocTestDefaults
+            sage: from sage.doctest.sources import FileDocTestSource
+            sage: filename = tmp_filename(ext='.py')
+            sage: s = "'''\n    sage: 2 + 2\n    4\n'''"
+            sage: with open(filename, 'w') as f:
+            ....:     _ = f.write(s)
+            sage: FDS = FileDocTestSource(filename, DocTestDefaults())
+            sage: for n, line in FDS:
+            ....:     print("{} {}".format(n, line))
+            0 '''
+            1     sage: 2 + 2
+            2     4
+            3 '''
+
+        The encoding is "utf-8" by default::
+
+            sage: FDS.encoding
+            'utf-8'
+
+        We create a file with a Latin-1 encoding without declaring it::
+
+            sage: s = b"'''\nRegardons le polyn\xF4me...\n'''\n"
+            sage: with open(filename, 'wb') as f:
+            ....:     _ = f.write(s)
+            sage: FDS = FileDocTestSource(filename, DocTestDefaults())
+            sage: L = list(FDS)
+            Traceback (most recent call last):
+            ...
+            UnicodeDecodeError: 'utf...8' codec can...t decode byte 0xf4 in position 18: invalid continuation byte
+
+        This works if we add a PEP 0263 encoding declaration::
+
+            sage: s = b"#!/usr/bin/env python\n# -*- coding: latin-1 -*-\n" + s
+            sage: with open(filename, 'wb') as f:
+            ....:     _ = f.write(s)
+            sage: FDS = FileDocTestSource(filename, DocTestDefaults())
+            sage: L = list(FDS)
+            sage: FDS.encoding
+            'latin-1'
+        """
+        with open(self.path, 'rb') as source:
+            for lineno, line in enumerate(source):
+                if lineno < 2:
+                    match = pep_0263.search(line)
+                    if match:
+                        self.encoding = bytes_to_str(match.group(1), 'ascii')
+                yield lineno, line.decode(self.encoding)
+
+    @lazy_attribute
+    def printpath(self):
+        """
+        Whether the path is printed absolutely or relatively depends on an option.
+
+        EXAMPLES::
+
+            sage: from sage.doctest.control import DocTestDefaults
+            sage: from sage.doctest.sources import FileDocTestSource
+            sage: import os
+            sage: filename = os.path.realpath(sage.doctest.sources.__file__)
+            sage: root = os.path.join(os.path.dirname(filename), '..')
+            sage: cwd = os.getcwd()
+            sage: os.chdir(root)
+            sage: FDS = FileDocTestSource(filename, DocTestDefaults(randorder=0,
+            ....:                                                   abspath=False))
+            sage: FDS.printpath
+            'doctest/sources.py'
+            sage: FDS = FileDocTestSource(filename, DocTestDefaults(randorder=0,
+            ....:                                                   abspath=True))
+            sage: FDS.printpath
+            '.../sage/doctest/sources.py'
+            sage: os.chdir(cwd)
+        """
+        if self.options.abspath:
+            return os.path.abspath(self.path)
+        else:
+            relpath = os.path.relpath(self.path)
+            if relpath.startswith(".." + os.path.sep):
+                return self.path
+            else:
+                return relpath
+
+    @lazy_attribute
+    def basename(self):
+        """
+        The basename of this file source, e.g. ``sage.doctest.sources``.
+
+        EXAMPLES::
+
+            sage: from sage.doctest.control import DocTestDefaults
+            sage: from sage.doctest.sources import FileDocTestSource
+            sage: filename = os.path.join(SAGE_SRC,'sage','rings','integer.pyx')
+            sage: FDS = FileDocTestSource(filename, DocTestDefaults())
+            sage: FDS.basename
+            'sage.rings.integer'
+        """
+        return get_basename(self.path)
+
+    @lazy_attribute
+    def in_lib(self):
+        """
+        Whether this file is to be treated as a module in a Python package.
+
+        Such files aren't loaded before running tests.
+
+        This uses :func:`~sage.misc.package_dir.is_package_or_sage_namespace_package_dir`
+        but can be overridden via :class:`~sage.doctest.control.DocTestDefaults`.
+
+        EXAMPLES::
+
+            sage: from sage.doctest.control import DocTestDefaults
+            sage: from sage.doctest.sources import FileDocTestSource
+            sage: import os
+            sage: filename = os.path.join(SAGE_SRC, 'sage', 'rings', 'integer.pyx')
+            sage: FDS = FileDocTestSource(filename, DocTestDefaults())
+            sage: FDS.in_lib
+            True
+            sage: filename = os.path.join(SAGE_SRC, 'sage', 'doctest', 'tests', 'abort.rst')
+            sage: FDS = FileDocTestSource(filename, DocTestDefaults())
+            sage: FDS.in_lib
+            False
+
+        You can override the default::
+
+            sage: FDS = FileDocTestSource("hello_world.py", DocTestDefaults())
+            sage: FDS.in_lib
+            False
+            sage: FDS = FileDocTestSource("hello_world.py", DocTestDefaults(force_lib=True))
+            sage: FDS.in_lib
+            True
+        """
+        return (self.options.force_lib
+                or is_package_or_sage_namespace_package_dir(os.path.dirname(self.path)))
+
+    @lazy_attribute
+    def file_optional_tags(self):
+        """
+        Return the set of tags that should apply to all doctests in this source.
+
+        EXAMPLES::
+
+            sage: from sage.doctest.control import DocTestDefaults
+            sage: from sage.doctest.sources import FileDocTestSource
+            sage: filename = sage.repl.user_globals.__file__
+            sage: FDS = FileDocTestSource(filename, DocTestDefaults())
+            sage: FDS.file_optional_tags
+            {'sage.modules': None}
+        """
+        from .parsing import parse_file_optional_tags
+        return parse_file_optional_tags(self)
+
+    def create_doctests(self, namespace):
+        r"""
+        Return a list of doctests for this file.
+
+        INPUT:
+
+        - ``namespace`` -- dictionary or :class:`sage.doctest.util.RecordingDict`
+
+        OUTPUT:
+
+        - ``doctests`` -- list of doctests defined in this file
+
+        - ``extras`` -- dictionary
+
+        EXAMPLES::
+
+            sage: from sage.doctest.control import DocTestDefaults
+            sage: from sage.doctest.sources import FileDocTestSource
+            sage: filename = sage.doctest.sources.__file__
+            sage: FDS = FileDocTestSource(filename, DocTestDefaults())
+            sage: doctests, extras = FDS.create_doctests(globals())
+            sage: len(doctests)
+            43
+            sage: extras['tab']
+            False
+
+        We give a self referential example::
+
+            sage: doctests[20].name
+            'sage.doctest.sources.FileDocTestSource.create_doctests'
+            sage: doctests[20].examples[8].source
+            'doctests[Integer(20)].examples[Integer(8)].source\n'
+
+        TESTS:
+
+        We check that we correctly process results that depend on 32
+        vs 64 bit architecture::
+
+            sage: import sys
+            sage: bitness = '64' if sys.maxsize > (1 << 32) else '32'
+            sage: gp.get_precision() == 38                                              # needs sage.libs.pari
+            False # 32-bit
+            True  # 64-bit
+            sage: ex = doctests[20].examples[11]
+            sage: ((bitness == '64' and ex.want == 'True  \n')                          # needs sage.libs.pari
+            ....:  or (bitness == '32' and ex.want == 'False \n'))
+            True
+
+        We check that lines starting with a # aren't doctested::
+
+            #sage: raise RuntimeError
+        """
+        if not os.path.exists(self.path):
+            import errno
+            raise OSError(errno.ENOENT, "File does not exist", self.path)
+        base, filename = os.path.split(self.path)
+        _, ext = os.path.splitext(filename)
+        if not self.in_lib and ext in ('.py', '.pyx', '.sage', '.spyx'):
+            cwd = os.getcwd()
+            if base:
+                os.chdir(base)
+            try:
+                load(filename, namespace) # errors raised here will be caught in DocTestTask
+            finally:
+                os.chdir(cwd)
+        self.qualified_name = NestedName(self.basename)
+        return self._create_doctests(namespace)
+
+    def _test_enough_doctests(self, check_extras=True, verbose=True):
+        r"""
+        This function checks to see that the doctests are not getting
+        unexpectedly skipped.
+
+        It uses a different (and simpler) code path than the doctest
+        creation functions. In particular, it does not understand
+        file-level and block-level # optional / needs tags.
+
+        INPUT:
+
+        - ``check_extras`` -- boolean (default: ``True``); whether to check if
+          doctests are created that do not correspond to either a ``sage:``
+          or a ``>>>`` prompt
+
+        - ``verbose`` -- boolean (default: ``True``); whether to print
+          offending line numbers when there are missing or extra tests
+
+        TESTS::
+
+            sage: # not tested (because the output will change when source files are changed)
+            sage: from sage.doctest.control import DocTestDefaults
+            sage: from sage.doctest.sources import FileDocTestSource
+            sage: cwd = os.getcwd()
+            sage: os.chdir(SAGE_SRC)
+            sage: import itertools
+            sage: for path, dirs, files in itertools.chain(os.walk('sage'), os.walk('doc')):
+            ....:     path = os.path.relpath(path)
+            ....:     dirs.sort(); files.sort()
+            ....:     for F in files:
+            ....:         _, ext = os.path.splitext(F)
+            ....:         if ext in ('.py', '.pyx', '.pxd', '.pxi', '.sage', '.spyx', '.rst'):
+            ....:             filename = os.path.join(path, F)
+            ....:             FDS = FileDocTestSource(filename, DocTestDefaults(long=True, optional=True, force_lib=True))
+            ....:             FDS._test_enough_doctests(verbose=False)
+            There are 4 unexpected tests being run in sage/doctest/parsing.py
+            There are 1 unexpected tests being run in sage/doctest/reporting.py
+            sage: os.chdir(cwd)
+        """
+        expected = []
+        rest = isinstance(self, RestSource)
+        if rest:
+            skipping = False
+            in_block = False
+            last_line = ''
+        for lineno, line in self:
+            if not line.strip():
+                continue
+            if rest:
+                if line.strip().startswith(".. nodoctest"):
+                    return
+                # We need to track blocks in order to figure out whether we're skipping.
+                if in_block:
+                    indent = whitespace.match(line).end()
+                    if indent <= starting_indent:
+                        in_block = False
+                        skipping = False
+                if not in_block:
+                    m1 = double_colon.match(line)
+                    m2 = code_block.match(line.lower())
+                    starting = (m1 and not line.strip().startswith(".. ")) or m2
+                    if starting:
+                        if ".. skip" in last_line:
+                            skipping = True
+                        in_block = True
+                        starting_indent = whitespace.match(line).end()
+                last_line = line
+            if (not rest or in_block) and sagestart.match(line) and not ((rest and skipping) or untested.search(line.lower())):
+                expected.append(lineno+1)
+        actual = []
+        tests, _ = self.create_doctests({})
+        for dt in tests:
+            if dt.examples:
+                for ex in dt.examples[:-1]: # the last entry is a sig_on_count()
+                    actual.append(dt.lineno + ex.lineno + 1)
+        shortfall = sorted(set(expected).difference(set(actual)))
+        extras = sorted(set(actual).difference(set(expected)))
+        if len(actual) == len(expected):
+            if not shortfall:
+                return
+            dif = extras[0] - shortfall[0]
+            for e, s in zip(extras[1:],shortfall[1:]):
+                if dif != e - s:
+                    break
+            else:
+                print("There are %s tests in %s that are shifted by %s" % (len(shortfall), self.path, dif))
+                if verbose:
+                    print("    The correct line numbers are %s" % (", ".join(str(n) for n in shortfall)))
+                return
+        elif len(actual) < len(expected):
+            print("There are %s tests in %s that are not being run" % (len(expected) - len(actual), self.path))
+        elif check_extras:
+            print("There are %s unexpected tests being run in %s" % (len(actual) - len(expected), self.path))
+        if verbose:
+            if shortfall:
+                print("    Tests on lines %s are not run" % (", ".join(str(n) for n in shortfall)))
+            if check_extras and extras:
+                print("    Tests on lines %s seem extraneous" % (", ".join(str(n) for n in extras)))
+
+
+class SourceLanguage:
+    """
+    An abstract class for functions that depend on the programming language of a doctest source.
+
+    Currently supported languages include Python, ReST and LaTeX.
+    """
+    def parse_docstring(self, docstring, namespace, start):
+        """
+        Return a list of doctest defined in this docstring.
+
+        This function is called by :meth:`DocTestSource._process_doc`.
+        The default implementation, defined here, is to use the
+        :class:`sage.doctest.parsing.SageDocTestParser` attached to
+        this source to get doctests from the docstring.
+
+        INPUT:
+
+        - ``docstring`` -- string containing documentation and tests
+
+        - ``namespace`` -- dictionary or :class:`sage.doctest.util.RecordingDict`
+
+        - ``start`` -- integer; one less than the starting line number
+
+        EXAMPLES::
+
+            sage: from sage.doctest.control import DocTestDefaults
+            sage: from sage.doctest.sources import FileDocTestSource
+            sage: from sage.doctest.parsing import SageDocTestParser
+            sage: from sage.doctest.util import NestedName
+            sage: filename = sage.doctest.util.__file__
+            sage: FDS = FileDocTestSource(filename, DocTestDefaults())
+            sage: doctests, _ = FDS.create_doctests({})
+            sage: for dt in doctests:
+            ....:     FDS.qualified_name = dt.name
+            ....:     dt.examples = dt.examples[:-1] # strip off the sig_on() test
+            ....:     assert(FDS.parse_docstring(dt.docstring,{},dt.lineno-1)[0] == dt)
+        """
+        return [self.parser.get_doctest(docstring, namespace, str(self.qualified_name),
+                                        self.printpath, start + 1)]
+
+
+class PythonSource(SourceLanguage):
+    """
+    This class defines the functions needed for the extraction of doctests from python sources.
+
+    EXAMPLES::
+
+        sage: from sage.doctest.control import DocTestDefaults
+        sage: from sage.doctest.sources import FileDocTestSource
+        sage: filename = sage.doctest.sources.__file__
+        sage: FDS = FileDocTestSource(filename, DocTestDefaults())
+        sage: type(FDS)
+        <class 'sage.doctest.sources.PythonFileSource'>
+    """
+    # The same line can't both start and end a docstring
+    start_finish_can_overlap = False
+
+    def _init(self):
+        """
+        This function is called before creating doctests from a Python source.
+
+        EXAMPLES::
+
+            sage: from sage.doctest.control import DocTestDefaults
+            sage: from sage.doctest.sources import FileDocTestSource
+            sage: filename = sage.doctest.sources.__file__
+            sage: FDS = FileDocTestSource(filename, DocTestDefaults())
+            sage: FDS._init()
+            sage: FDS.last_indent
+            -1
+        """
+        self.last_indent = -1
+        self.last_line = None
+        self.quotetype = None
+        self.paren_count = 0
+        self.bracket_count = 0
+        self.curly_count = 0
+        self.code_wrapping = False
+
+    def _update_quotetype(self, line):
+        r"""
+        Update the track of what kind of quoted string we are in.
+
+        We need to track whether we're inside a triple quoted
+        string, since a triple quoted string that starts a line
+        could be the end of a string and thus not the beginning of a
+        doctest (see sage.misc.sageinspect for an example).
+
+        To do this tracking we need to track whether we're inside a
+        string at all, since ''' inside a string doesn't start a
+        triple quote (see the top of this file for an example).
+
+        We also need to track parentheses and brackets, since we only
+        want to update our record of last line and indentation level
+        when the line is actually over.
+
+        EXAMPLES::
+
+            sage: from sage.doctest.control import DocTestDefaults
+            sage: from sage.doctest.sources import FileDocTestSource
+            sage: filename = sage.doctest.sources.__file__
+            sage: FDS = FileDocTestSource(filename, DocTestDefaults())
+            sage: FDS._init()
+            sage: FDS._update_quotetype('\"\"\"'); print(" ".join(list(FDS.quotetype)))
+            " " "
+            sage: FDS._update_quotetype("'''"); print(" ".join(list(FDS.quotetype)))
+            " " "
+            sage: FDS._update_quotetype('\"\"\"'); print(FDS.quotetype)
+            None
+            sage: FDS._update_quotetype("triple_quotes = re.compile(\"\\s*[rRuU]*((''')|(\\\"\\\"\\\"))\")")
+            sage: print(FDS.quotetype)
+            None
+            sage: FDS._update_quotetype("''' Single line triple quoted string \\''''")
+            sage: print(FDS.quotetype)
+            None
+            sage: FDS._update_quotetype("' Lots of \\\\\\\\'")
+            sage: print(FDS.quotetype)
+            None
+        """
+        def _update_parens(start, end=None):
+            self.paren_count += line.count("(",start,end) - line.count(")",start,end)
+            self.bracket_count += line.count("[",start,end) - line.count("]",start,end)
+            self.curly_count += line.count("{",start,end) - line.count("}",start,end)
+        pos = 0
+        while pos < len(line):
+            if self.quotetype is None:
+                next_single = line.find("'",pos)
+                next_double = line.find('"',pos)
+                if next_single == -1 and next_double == -1:
+                    next_comment = line.find("#",pos)
+                    if next_comment == -1:
+                        _update_parens(pos)
+                    else:
+                        _update_parens(pos,next_comment)
+                    break
+                elif next_single == -1:
+                    m = next_double
+                elif next_double == -1:
+                    m = next_single
+                else:
+                    m = min(next_single, next_double)
+                next_comment = line.find('#',pos,m)
+                if next_comment != -1:
+                    _update_parens(pos,next_comment)
+                    break
+                _update_parens(pos,m)
+                if m+2 < len(line) and line[m] == line[m+1] == line[m+2]:
+                    self.quotetype = line[m:m+3]
+                    pos = m+3
+                else:
+                    self.quotetype = line[m]
+                    pos = m+1
+            else:
+                next = line.find(self.quotetype,pos)
+                if next == -1:
+                    break
+                elif next == 0 or line[next-1] != '\\':
+                    pos = next + len(self.quotetype)
+                    self.quotetype = None
+                else:
+                    # We need to worry about the possibility that
+                    # there are an even number of backslashes before
+                    # the quote, in which case it is not escaped
+                    count = 1
+                    slashpos = next - 2
+                    while slashpos >= pos and line[slashpos] == '\\':
+                        count += 1
+                        slashpos -= 1
+                    if count % 2 == 0:
+                        pos = next + len(self.quotetype)
+                        self.quotetype = None
+                    else:
+                        # The possible ending quote was escaped.
+                        pos = next + 1
+
+    def starting_docstring(self, line):
+        """
+        Determines whether the input line starts a docstring.
+
+        If the input line does start a docstring (a triple quote),
+        then this function updates ``self.qualified_name``.
+
+        INPUT:
+
+        - ``line`` -- string; one line of an input file
+
+        OUTPUT: either ``None`` or a Match object
+
+        EXAMPLES::
+
+            sage: from sage.doctest.control import DocTestDefaults
+            sage: from sage.doctest.sources import FileDocTestSource
+            sage: from sage.doctest.util import NestedName
+            sage: filename = sage.doctest.sources.__file__
+            sage: FDS = FileDocTestSource(filename, DocTestDefaults())
+            sage: FDS._init()
+            sage: FDS.starting_docstring("r'''")
+            <...Match object...>
+            sage: FDS.ending_docstring("'''")
+            <...Match object...>
+            sage: FDS.qualified_name = NestedName(FDS.basename)
+            sage: FDS.starting_docstring("class MyClass():")
+            sage: FDS.starting_docstring("    def hello_world(self):")
+            sage: FDS.starting_docstring("        '''")
+            <...Match object...>
+            sage: FDS.qualified_name
+            sage.doctest.sources.MyClass.hello_world
+            sage: FDS.ending_docstring("    '''")
+            <...Match object...>
+            sage: FDS.starting_docstring("class NewClass():")
+            sage: FDS.starting_docstring("    '''")
+            <...Match object...>
+            sage: FDS.ending_docstring("    '''")
+            <...Match object...>
+            sage: FDS.qualified_name
+            sage.doctest.sources.NewClass
+            sage: FDS.starting_docstring("print(")
+            sage: FDS.starting_docstring("    '''Not a docstring")
+            sage: FDS.starting_docstring("    ''')")
+            sage: FDS.starting_docstring("def foo():")
+            sage: FDS.starting_docstring("    '''This is a docstring'''")
+            <...Match object...>
+        """
+        indent = whitespace.match(line).end()
+        quotematch = None
+        if self.quotetype is None and not self.code_wrapping:
+            # We're not inside a triple quote and not inside code like
+            # print(
+            #     """Not a docstring
+            #     """)
+
+            if line[indent] != '#' and (indent == 0 or indent > self.last_indent):
+                quotematch = triple_quotes.match(line)
+                # It would be nice to only run the name_regex when
+                # quotematch wasn't None, but then we mishandle classes
+                # that don't have a docstring.
+                if not self.code_wrapping and self.last_indent >= 0 and indent > self.last_indent:
+                    name = name_regex.match(self.last_line)
+                    if name:
+                        name = name.groups()[0]
+                        self.qualified_name[indent] = name
+                    elif quotematch:
+                        self.qualified_name[indent] = '?'
+        self._update_quotetype(line)
+        if line[indent] != '#' and not self.code_wrapping:
+            self.last_line, self.last_indent = line, indent
+        self.code_wrapping = not (self.paren_count == self.bracket_count == self.curly_count == 0)
+        return quotematch
+
+    def ending_docstring(self, line):
+        r"""
+        Determines whether the input line ends a docstring.
+
+        INPUT:
+
+        - ``line`` -- string, one line of an input file
+
+        OUTPUT: an object that, when evaluated in a boolean context, gives
+        ``True`` or ``False`` depending on whether the input line marks the
+        end of a docstring
+
+        EXAMPLES::
+
+            sage: from sage.doctest.control import DocTestDefaults
+            sage: from sage.doctest.sources import FileDocTestSource
+            sage: from sage.doctest.util import NestedName
+            sage: filename = sage.doctest.sources.__file__
+            sage: FDS = FileDocTestSource(filename, DocTestDefaults())
+            sage: FDS._init()
+            sage: FDS.quotetype = "'''"
+            sage: FDS.ending_docstring("'''")
+            <...Match object...>
+            sage: FDS.ending_docstring('\"\"\"')
+        """
+        quotematch = triple_quotes.match(line)
+        if quotematch is not None and quotematch.groups()[0] != self.quotetype:
+            quotematch = None
+        self._update_quotetype(line)
+        return quotematch
+
+    def _neutralize_doctests(self, reindent):
+        r"""
+        Return a string containing the source of ``self``, but with
+        doctests modified so they are not tested.
+
+        This function is used in creating doctests for ReST files,
+        since docstrings of Python functions defined inside verbatim
+        blocks screw up Python's doctest parsing.
+
+        INPUT:
+
+        - ``reindent`` -- integer; the number of spaces to indent
+          the result
+
+        EXAMPLES::
+
+            sage: from sage.doctest.control import DocTestDefaults
+            sage: from sage.doctest.sources import StringDocTestSource, PythonSource
+            sage: from sage.structure.dynamic_class import dynamic_class
+            sage: s = "'''\n    sage: 2 + 2\n    4\n'''"
+            sage: PythonStringSource = dynamic_class('PythonStringSource',
+            ....:                                    (StringDocTestSource, PythonSource))
+            sage: PSS = PythonStringSource('<runtime>', s, DocTestDefaults(), 'runtime')
+            sage: print(PSS._neutralize_doctests(0))
+            '''
+                safe: 2 + 2
+                4
+            '''
+        """
+        neutralized = []
+        in_docstring = False
+        self._init()
+        for lineno, line in self:
+            if not line.strip():
+                neutralized.append(line)
+            elif in_docstring:
+                if self.ending_docstring(line):
+                    in_docstring = False
+                neutralized.append(" "*reindent + find_prompt.sub(r"\1safe:\3",line))
+            else:
+                if self.starting_docstring(line):
+                    in_docstring = True
+                neutralized.append(" "*reindent + line)
+        return "".join(neutralized)
+
+
+class TexSource(SourceLanguage):
+    """
+    This class defines the functions needed for the extraction of
+    doctests from a LaTeX source.
+
+    EXAMPLES::
+
+        sage: from sage.doctest.control import DocTestDefaults
+        sage: from sage.doctest.sources import FileDocTestSource
+        sage: filename = "sage_paper.tex"
+        sage: FDS = FileDocTestSource(filename, DocTestDefaults())
+        sage: type(FDS)
+        <class 'sage.doctest.sources.TexFileSource'>
+    """
+    # The same line can't both start and end a docstring
+    start_finish_can_overlap = False
+
+    def _init(self):
+        """
+        This function is called before creating doctests from a Tex file.
+
+        EXAMPLES::
+
+            sage: from sage.doctest.control import DocTestDefaults
+            sage: from sage.doctest.sources import FileDocTestSource
+            sage: filename = "sage_paper.tex"
+            sage: FDS = FileDocTestSource(filename, DocTestDefaults())
+            sage: FDS._init()
+            sage: FDS.skipping
+            False
+        """
+        self.skipping = False
+
+    def starting_docstring(self, line):
+        r"""
+        Determines whether the input line starts a docstring.
+
+        Docstring blocks in tex files are defined by verbatim or
+        lstlisting environments, and can be linked together by adding
+        %link immediately after the \end{verbatim} or \end{lstlisting}.
+
+        Within a verbatim (or lstlisting) block, you can tell Sage not to
+        process the rest of the block by including a %skip line.
+
+        INPUT:
+
+        - ``line`` -- string, one line of an input file
+
+        OUTPUT: boolean; whether the input line marks the start of a docstring
+        (verbatim block)
+
+        EXAMPLES::
+
+            sage: from sage.doctest.control import DocTestDefaults
+            sage: from sage.doctest.sources import FileDocTestSource
+            sage: filename = "sage_paper.tex"
+            sage: FDS = FileDocTestSource(filename, DocTestDefaults())
+            sage: FDS._init()
+
+        We start docstrings with \begin{verbatim} or \begin{lstlisting}::
+
+            sage: FDS.starting_docstring(r"\begin{verbatim}")
+            True
+            sage: FDS.starting_docstring(r"\begin{lstlisting}")
+            True
+            sage: FDS.skipping
+            False
+            sage: FDS.ending_docstring("sage: 2+2")
+            False
+            sage: FDS.ending_docstring("4")
+            False
+
+        To start ignoring the rest of the verbatim block, use %skip::
+
+            sage: FDS.ending_docstring("%skip")
+            True
+            sage: FDS.skipping
+            True
+            sage: FDS.starting_docstring("sage: raise RuntimeError")
+            False
+
+        You can even pretend to start another verbatim block while skipping::
+
+            sage: FDS.starting_docstring(r"\begin{verbatim}")
+            False
+            sage: FDS.skipping
+            True
+
+        To stop skipping end the verbatim block::
+
+            sage: FDS.starting_docstring(r"\end{verbatim} %link")
+            False
+            sage: FDS.skipping
+            False
+
+        Linking works even when the block was ended while skipping::
+
+            sage: FDS.linking
+            True
+            sage: FDS.starting_docstring(r"\begin{verbatim}")
+            True
+        """
+        if self.skipping:
+            if self.ending_docstring(line, check_skip=False):
+                self.skipping = False
+            return False
+        return bool(begin_verb.match(line) or begin_lstli.match(line))
+
+    def ending_docstring(self, line, check_skip=True):
+        r"""
+        Determines whether the input line ends a docstring.
+
+        Docstring blocks in tex files are defined by verbatim or
+        lstlisting environments, and can be linked together by adding
+        %link immediately after the \end{verbatim} or \end{lstlisting}.
+
+        Within a verbatim (or lstlisting) block, you can tell Sage not to
+        process the rest of the block by including a %skip line.
+
+        INPUT:
+
+        - ``line`` -- string, one line of an input file
+
+        - ``check_skip`` -- boolean (default: ``True``); used internally in
+          ``starting_docstring``
+
+        OUTPUT: boolean; whether the input line marks the end of a docstring
+        (verbatim block)
+
+        EXAMPLES::
+
+            sage: from sage.doctest.control import DocTestDefaults
+            sage: from sage.doctest.sources import FileDocTestSource
+            sage: filename = "sage_paper.tex"
+            sage: FDS = FileDocTestSource(filename, DocTestDefaults())
+            sage: FDS._init()
+            sage: FDS.ending_docstring(r"\end{verbatim}")
+            True
+            sage: FDS.ending_docstring(r"\end{lstlisting}")
+            True
+            sage: FDS.linking
+            False
+
+        Use %link to link with the next verbatim block::
+
+            sage: FDS.ending_docstring(r"\end{verbatim}%link")
+            True
+            sage: FDS.linking
+            True
+
+        %skip also ends a docstring block::
+
+            sage: FDS.ending_docstring("%skip")
+            True
+        """
+        m = end_verb.match(line)
+        if m:
+            if m.groups()[0]:
+                self.linking = True
+            else:
+                self.linking = False
+            return True
+        m = end_lstli.match(line)
+        if m:
+            if m.groups()[0]:
+                self.linking = True
+            else:
+                self.linking = False
+            return True
+        if check_skip and skip.match(line):
+            self.skipping = True
+            return True
+        return False
+
+
+class RestSource(SourceLanguage):
+    """
+    This class defines the functions needed for the extraction of
+    doctests from ReST sources.
+
+    EXAMPLES::
+
+        sage: from sage.doctest.control import DocTestDefaults
+        sage: from sage.doctest.sources import FileDocTestSource
+        sage: filename = "sage_doc.rst"
+        sage: FDS = FileDocTestSource(filename, DocTestDefaults())
+        sage: type(FDS)
+        <class 'sage.doctest.sources.RestFileSource'>
+    """
+    # The same line can both start and end a docstring
+    start_finish_can_overlap = True
+
+    def _init(self):
+        """
+        This function is called before creating doctests from a ReST file.
+
+        EXAMPLES::
+
+            sage: from sage.doctest.control import DocTestDefaults
+            sage: from sage.doctest.sources import FileDocTestSource
+            sage: filename = "sage_doc.rst"
+            sage: FDS = FileDocTestSource(filename, DocTestDefaults())
+            sage: FDS._init()
+            sage: FDS.link_all
+            False
+        """
+        self.link_all = False
+        self.last_line = ""
+        self.last_indent = -1
+        self.first_line = False
+        self.skipping = False
+
+    def starting_docstring(self, line):
+        """
+        A line ending with a double colon starts a verbatim block in a ReST file,
+        as does a line containing ``.. CODE-BLOCK:: language``.
+
+        This function also determines whether the docstring block
+        should be joined with the previous one, or should be skipped.
+
+        INPUT:
+
+        - ``line`` -- string; one line of an input file
+
+        OUTPUT: either ``None`` or a Match object
+
+        EXAMPLES::
+
+            sage: from sage.doctest.control import DocTestDefaults
+            sage: from sage.doctest.sources import FileDocTestSource
+            sage: filename = "sage_doc.rst"
+            sage: FDS = FileDocTestSource(filename, DocTestDefaults())
+            sage: FDS._init()
+            sage: FDS.starting_docstring("Hello world::")
+            True
+            sage: FDS.ending_docstring("    sage: 2 + 2")
+            False
+            sage: FDS.ending_docstring("    4")
+            False
+            sage: FDS.ending_docstring("We are now done")
+            True
+            sage: FDS.starting_docstring(".. link")
+            sage: FDS.starting_docstring("::")
+            True
+            sage: FDS.linking
+            True
+        """
+        if link_all.match(line):
+            self.link_all = True
+        if self.skipping:
+            end_block = self.ending_docstring(line)
+            if end_block:
+                self.skipping = False
+            else:
+                return False
+        m1 = double_colon.match(line)
+        m2 = code_block.match(line.lower())
+        starting = (m1 and not line.strip().startswith(".. ")) or m2
+        if starting:
+            self.linking = self.link_all or '.. link' in self.last_line
+            self.first_line = True
+            m = m1 or m2
+            indent = len(m.groups()[0])
+            if '.. skip' in self.last_line:
+                self.skipping = True
+                starting = False
+        else:
+            indent = self.last_indent
+        self.last_line, self.last_indent = line, indent
+        return starting
+
+    def ending_docstring(self, line):
+        """
+        When the indentation level drops below the initial level the
+        block ends.
+
+        INPUT:
+
+        - ``line`` -- string; one line of an input file
+
+        OUTPUT: boolean; whether the verbatim block is ending
+
+        EXAMPLES::
+
+            sage: from sage.doctest.control import DocTestDefaults
+            sage: from sage.doctest.sources import FileDocTestSource
+            sage: filename = "sage_doc.rst"
+            sage: FDS = FileDocTestSource(filename, DocTestDefaults())
+            sage: FDS._init()
+            sage: FDS.starting_docstring("Hello world::")
+            True
+            sage: FDS.ending_docstring("    sage: 2 + 2")
+            False
+            sage: FDS.ending_docstring("    4")
+            False
+            sage: FDS.ending_docstring("We are now done")
+            True
+        """
+        if not line.strip():
+            return False
+        indent = whitespace.match(line).end()
+        if self.first_line:
+            self.first_line = False
+            if indent <= self.last_indent:
+                # We didn't indent at all
+                return True
+            self.last_indent = indent
+        return indent < self.last_indent
+
+    def parse_docstring(self, docstring, namespace, start):
+        r"""
+        Return a list of doctest defined in this docstring.
+
+        Code blocks in a REST file can contain python functions with
+        their own docstrings in addition to in-line doctests.  We want
+        to include the tests from these inner docstrings, but Python's
+        doctesting module has a problem if we just pass on the whole
+        block, since it expects to get just a docstring, not the
+        Python code as well.
+
+        Our solution is to create a new doctest source from this code
+        block and append the doctests created from that source.  We
+        then replace the occurrences of "sage:" and ">>>" occurring
+        inside a triple quote with "safe:" so that the doctest module
+        doesn't treat them as tests.
+
+        EXAMPLES::
+
+            sage: from sage.doctest.control import DocTestDefaults
+            sage: from sage.doctest.sources import FileDocTestSource
+            sage: from sage.doctest.parsing import SageDocTestParser
+            sage: from sage.doctest.util import NestedName
+            sage: filename = "sage_doc.rst"
+            sage: FDS = FileDocTestSource(filename, DocTestDefaults())
+            sage: FDS.parser = SageDocTestParser(set(['sage']))
+            sage: FDS.qualified_name = NestedName('sage_doc')
+            sage: s = "Some text::\n\n    def example_python_function(a, \
+            ....:      b):\n        '''\n        Brief description \
+            ....:      of function.\n\n        EXAMPLES::\n\n            \
+            ....:      sage: test1()\n            sage: test2()\n        \
+            ....:      '''\n        return a + b\n\n    sage: test3()\n\nMore \
+            ....:      ReST documentation."
+            sage: tests = FDS.parse_docstring(s, {}, 100)
+            sage: len(tests)
+            2
+            sage: for ex in tests[0].examples:
+            ....:     print(ex.sage_source)
+            test3()
+            sage: for ex in tests[1].examples:
+            ....:     print(ex.sage_source)
+            test1()
+            test2()
+            sig_on_count() # check sig_on/off pairings (virtual doctest)
+        """
+        PythonStringSource = dynamic_class("sage.doctest.sources.PythonStringSource",
+                                           (StringDocTestSource, PythonSource))
+        min_indent = self.parser._min_indent(docstring)
+        pysource = '\n'.join(l[min_indent:] for l in docstring.split('\n'))
+        inner_source = PythonStringSource(self.basename, pysource,
+                                          self.options,
+                                          self.printpath,
+                                          lineno_shift=start + 1)
+        inner_doctests, _ = inner_source._create_doctests(namespace, True)
+        safe_docstring = inner_source._neutralize_doctests(min_indent)
+        outer_doctest = self.parser.get_doctest(safe_docstring, namespace,
+                                                str(self.qualified_name),
+                                                self.printpath, start + 1)
+        return [outer_doctest] + inner_doctests
+
+
+class DictAsObject(dict):
+    """
+    A simple subclass of dict that inserts the items from the initializing dictionary into attributes.
+
+    EXAMPLES::
+
+        sage: from sage.doctest.sources import DictAsObject
+        sage: D = DictAsObject({'a':2})
+        sage: D.a
+        2
+    """
+    def __init__(self, attrs):
+        """
+        Initialization.
+
+        INPUT:
+
+        - ``attrs`` -- dictionary
+
+        EXAMPLES::
+
+            sage: from sage.doctest.sources import DictAsObject
+            sage: D = DictAsObject({'a':2})
+            sage: D.a == D['a']
+            True
+            sage: D.a
+            2
+        """
+        super().__init__(attrs)
+        self.__dict__.update(attrs)
+
+    def __setitem__(self, ky, val):
+        """
+        We preserve the ability to access entries through either the
+        dictionary or attribute interfaces.
+
+        EXAMPLES::
+
+            sage: from sage.doctest.sources import DictAsObject
+            sage: D = DictAsObject({})
+            sage: D['a'] = 2
+            sage: D.a
+            2
+        """
+        super().__setitem__(ky, val)
+        try:
+            super().__setattr__(ky, val)
+        except TypeError:
+            pass
+
+    def __setattr__(self, ky, val):
+        """
+        We preserve the ability to access entries through either the
+        dictionary or attribute interfaces.
+
+        EXAMPLES::
+
+            sage: from sage.doctest.sources import DictAsObject
+            sage: D = DictAsObject({})
+            sage: D.a = 2
+            sage: D['a']
+            2
+        """
+        super().__setitem__(ky, val)
+        super().__setattr__(ky, val)